风暴，HDFSBolt

时间：2016-02-04 14:55:58

标签： hdfs apache-storm

我正在尝试在HDFSBolt中实施Storm。我想从基本的开始，所以使用TestWordSpout提供的Storm。

我可以成功编译topology，但是当我提交它时，我收到以下错误

8102 [Thread-17-output-executor[3 3]] ERROR o.a.s.util - Async loop died!
java.lang.NoClassDefFoundError: org/apache/hadoop/fs/CanUnbuffer
        at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.7.0_95]
        .......
        .......

这是我的topology 

public class HdfsFileTopology {
  public static void main(String[] args) throws Exception {
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(",");
    SyncPolicy syncPolicy = new CountSyncPolicy(100);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(10.0f, Units.KB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/user");
    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl("hdfs://localhost:9000")
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 1);
    builder.setBolt("output", bolt, 1).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setNumWorkers(3);
    StormSubmitter.submitTopology("HdfsFileTopology", conf, builder.createTopology());
    }
}

这是我的pom.xml 

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
  <artifactId>storm</artifactId>
  <groupId>org.apache.storm</groupId>
  <version>1.0.0-SNAPSHOT</version>
  <relativePath>../../pom.xml</relativePath>
</parent>

<groupId>org.apache.storm</groupId>
<artifactId>storm-test</artifactId>
<packaging>jar</packaging>

<name>storm-test</name>

<properties>
   <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   <provided.scope>provided</provided.scope>
</properties>
<profiles>
  <profile>
    <id>intellij</id>
    <properties>
        <provided.scope>compile</provided.scope>
    </properties>
  </profile>
</profiles>

<dependencies>
  <dependency>
    <groupId>org.hdrhistogram</groupId>
    <artifactId>HdrHistogram</artifactId>
  </dependency>
  <dependency>
    <groupId>junit</groupId>
    <artifactId>junit</artifactId>
    <scope>test</scope>
  </dependency>
  <dependency>
    <groupId>org.testng</groupId>
    <artifactId>testng</artifactId>
    <version>6.8.5</version>
    <scope>test</scope>
  </dependency>
  <dependency>
    <groupId>org.mockito</groupId>
    <artifactId>mockito-all</artifactId>
    <scope>test</scope>
  </dependency>
  <dependency>
    <groupId>org.easytesting</groupId>
    <artifactId>fest-assert-core</artifactId>
    <version>2.0M8</version>
    <scope>test</scope>
  </dependency>
  <dependency>
    <groupId>org.jmock</groupId>
    <artifactId>jmock</artifactId>
    <version>2.6.0</version>
    <scope>test</scope>
  </dependency>
  <dependency>
   <groupId>org.twitter4j</groupId>
   <artifactId>twitter4j-stream</artifactId>
   <version>3.0.3</version>
   </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-core</artifactId>
    <version>${project.version}</version>
    <scope>${provided.scope}</scope>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-core</artifactId>
    <version>${project.version}</version>
    <type>test-jar</type>
    <scope>test</scope>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>multilang-javascript</artifactId>
    <version>${project.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>multilang-ruby</artifactId>
    <version>${project.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>multilang-python</artifactId>
    <version>${project.version}</version>
  </dependency>
  <dependency>
    <groupId>commons-collections</groupId>
    <artifactId>commons-collections</artifactId>
    <version>3.2.1</version>
  </dependency>
  <dependency>
    <groupId>com.google.guava</groupId>
    <artifactId>guava</artifactId>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-metrics</artifactId>
    <version>${project.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-kafka</artifactId>
    <version>${project.version}</version>
    <scope>provided</scope>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-hdfs</artifactId>
    <version>${project.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka_2.10</artifactId>
    <version>0.8.2.1</version>
    <scope>provided</scope>
    <exclusions>
      <exclusion>
        <groupId>org.apache.zookeeper</groupId>
        <artifactId>zookeeper</artifactId>
      </exclusion>
      <exclusion>
        <groupId>log4j</groupId>
        <artifactId>log4j</artifactId>
      </exclusion>
    </exclusions>
  </dependency>
  <dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>0.8.2.1</version>
    <scope>provided</scope>
  </dependency>
  <dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-redis</artifactId>
    <version>${project.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.7.1</version>
    <exclusions>
      <exclusion>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
      </exclusion>
    </exclusions>
  </dependency>
  <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.7.1</version>
    <exclusions>
      <exclusion>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
      </exclusion>
    </exclusions>
  </dependency>
</dependencies>

<build>
<sourceDirectory>src/jvm</sourceDirectory>
<testSourceDirectory>test/jvm</testSourceDirectory>
<resources>
  <resource>
    <directory>${basedir}/multilang</directory>
  </resource>
</resources>

<plugins>
    <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <configuration>
            <createDependencyReducedPom>true</createDependencyReducedPom>
        </configuration>
        <executions>
            <execution>
                <phase>package</phase>
                <goals>
                    <goal>shade</goal>
                </goals>
                <configuration>
                    <transformers>
                        <transformer
                                implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                        <transformer
                                implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                <mainClass></mainClass>
                        </transformer>
                    </transformers>
                </configuration>
            </execution>
        </executions>
    </plugin>

  <plugin>
    <groupId>com.theoryinpractise</groupId>
    <artifactId>clojure-maven-plugin</artifactId>
    <extensions>true</extensions>
    <configuration>
      <sourceDirectories>
        <sourceDirectory>src/clj</sourceDirectory>
      </sourceDirectories>
    </configuration>
    <executions>
      <execution>
        <id>compile</id>
        <phase>compile</phase>
        <goals>
          <goal>compile</goal>
        </goals>
      </execution>
    </executions>
  </plugin>

  <plugin>
    <groupId>org.codehaus.mojo</groupId>
    <artifactId>exec-maven-plugin</artifactId>
    <version>1.2.1</version>
    <executions>
      <execution>
        <goals>
          <goal>exec</goal>
        </goals>
      </execution>
    </executions>
    <configuration>
      <executable>java</executable>
      <includeProjectDependencies>true</includeProjectDependencies>
      <includePluginDependencies>false</includePluginDependencies>
      <classpathScope>compile</classpathScope>
      <mainClass>${storm.topology}</mainClass>
    </configuration>
  </plugin>
</plugins>

2 个答案:

答案 0 :(得分：1)

当我的拓扑构建在带有版本的风暴库时，我有很多NoClassDefFoundError，这与我在集群上的不同。确保使用正确的版本。

答案 1 :(得分：1)

听起来像是一个罐子包装问题。确保你的jar包含必需的依赖项（只需要procedure TfrmNocoreDKS.FormCreate(Sender: TObject); begin Inherited; System.SysUtils.FormatSettings.ShortDateFormat := 'dd/mm/yyyy'; CheckPhone; ConnectfromProfile; if not Assigned(fProfileAction) then ConnectDatabase Else lstDocuments.Enabled := False; {$IFDEF ANDROID} ChangeComboBoxStyle; {$ENDIF} end; procedure TfrmNocoreDKS.ConnectfromProfile; begin fdmProfileConnection := TdmConnection.Create(nil); fdmProfileConnection.OpenProfileDb; fdmProfileConnection.LoadProfiles; if fdmProfileConnection.Profiles.Count = 0 then begin // Createdefault Profile fProfileAction := actCreateNewProfileExecute; end else if fdmProfileConnection.Profiles.Count = 1 then begin // one profile load connection; fProfileAction := nil; fCurrentProfile := fdmProfileConnection.Profiles.Items[0]; end else begin // multiple profiles choose connection; fProfileAction := SelectProfileOnStartUp; end; end; procedure TfrmNocoreDKS.FormShow(Sender: TObject); begin if Assigned(fProfileAction) then fProfileAction(Self); end; procedure TfrmNocoreDKS.actCreateNewProfileExecute(Sender: TObject); var profilename, databasename, pathname: string; prf: TfrmProfile; begin prf := TfrmProfile.Create(nil); prf.Data := fdmProfileConnection.Profiles; prf.ShowModal( procedure(ModalResult: TModalResult) begin if ModalResult = mrOk then begin profilename := prf.edtProfilename.Text; databasename := prf.edtDatabaseName.Text; {$IFDEF IOS} pathname := System.IOUtils.TPath.GetDocumentsPath; {$ENDIF} {$IFDEF ANDROID} pathname := System.IOUtils.TPath.GetDocumentsPath; {$ENDIF} {$IFDEF WIN32} pathname := ExtractFilePath(ParamStr(0)) + '\Data'; {$ENDIF} FDSQLiteBackup1.Database := System.IOUtils.TPath.Combine(pathname, 'default.sqlite3'); // Default Database FDSQLiteBackup1.DestDatabase := System.IOUtils.TPath.Combine(pathname, databasename + '.sqlite3'); FDSQLiteBackup1.Backup; fdmProfileConnection.AddProfile(databasename + '.sqlite3', profilename); fdmProfileConnection.LoadProfiles; fCurrentProfile := fdmProfileConnection.Profiles.Items[0]; connectDatabase; end else Application.Terminate; end); end; 中的依赖项并不一定保证你的jar包含它们。）

我通常将pom.xml与maven-jar-plugin结合使用来组装jar。请看这里的示例：https://github.com/mjsax/aeolus/blob/master/queries/lrb/pom.xml

maven-dependency-plugin中的<includes>部分列出了其他依赖项。 maven-jar-plugin使这些相关文件可用（例如maven-dependency-plugin）。

据我所知，commons-lang3也可用于组装jar（阅读文档以获取更多信息）。

相关问题
最新问题