Ubuntu 14.04上的Hadoop 2.7.1和Eclipse本地运行:java.lang.NoClassDefFoundError

时间:2015-07-13 20:08:34

标签: java eclipse maven hadoop noclassdeffounderror

我最近开始学习如何设置Hadoop,并按照Hadoop:The Definitive Guide一书编写了我的第一个应用程序。但是,我陷入了一个必须在一小段数据上本地运行我的应用程序的步骤。

mvn compile

export HADOOP_CLASSPATH=~/workspace/hadoop-book-mr-dev/target/classes/com/hadoopbook/hadoop_book_mr_dev/

hadoop MaxTemperatureDriver -conf conf/hadoop-local.xml input/ncdc/micro output

我在命令行中运行了上面的命令,并导出了包含所需类的Eclipse工作区$HADOOP_CLASSPATH。使用我的应用程序分析数据的最后一个命令没有成功,并给我这个错误:

Exception in thread "main" java.lang.NoClassDefFoundError: MaxTemperatureDriver (wrong name: com/hadoopbook/hadoop_book_mr_dev/MaxTemperatureDriver)
    at java.lang.ClassLoader.defineClass1(Native Method)
    at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
    at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
    at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:482)

我尝试过不同的文件夹和目录作为我的类路径,但无济于事。另外,我读了其他类似的主题并做Clean->Clean Selected Projects以及从我的构建路径中删除源文件夹,然后再将其添加回来,但它们都不适用于我。

最后,我还检查了我的Eclipse中有commons-logging-1.1.1.jarhadoop-yarn-common-2.7.1.jar作为buildpath。我不知道此刻还有什么可以尝试的,如果有人能够指导我逐步完成这一步,我将不胜感激。

以下是一些有助于调试问题的其他信息。 mvn eclipse步骤构建的pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.hadoopbook</groupId>
  <artifactId>hadoop-book-mr-dev</artifactId>
  <version>4.0</version>


  <name>hadoop-book-mr-dev</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <hadoop.version>2.7.1</hadoop.version>
  </properties>

  <dependencies>
    <!-- Hadoop main client artifact -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>${hadoop.version}</version>

    </dependency>

    <!-- Unit test artifact -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>

    <dependency>
        <groupId>org.apache.mrunit</groupId>
        <artifactId>mrunit</artifactId>
        <version>1.1.0</version>
        <classifier>hadoop2</classifier>
        <scope>test</scope>
    </dependency>

    <!-- Hadoop test artifact for running mini clusters -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-minicluster</artifactId>
        <version>${hadoop.version}</version>
        <scope>test</scope>
    </dependency>
  </dependencies>
  <build>
    <finalName>hadoop-examples</finalName>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
            <configuration>
                <source>1.6</source>
                <target>1.6</target>
            </configuration>

        </plugin>

        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-jar-plugins</artifactId>
            <version>2.5</version>
            <configuration>
                <outputDirectory>${basedir}</outputDirectory>
            </configuration>
        </plugin>
    </plugins>
  </build>
</project>

hadoop命令尝试运行的Driver类:

package com.hadoopbook.hadoop_book_mr_dev;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class MaxTemperatureDriver extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {
    if(args.length != 2){
        System.err.printf("Usage: $s [generic options] <input> <output> \n",
                getClass().getSimpleName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;

        }


    Job job = new Job(getConf(), "Max temperature");
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job,  new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(MaxTemperatureMapper.class);
    job.setCombinerClass(MaxTemperatureReducer.class);
    job.setReducerClass(MaxTemperatureReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
    }

    public static void main(String[] args) throws Exception {
        int exitCode = ToolRunner.run(new MaxTemperatureDriver(), args);
        System.exit(exitCode);

    }
}

我的构建路径中的JAR和Maven依赖项列表:

/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-client/2.7.1/hadoop-client-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-common/2.7.1/hadoop-common-2.7.1.jar
/home/ubuntu/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar
/home/ubuntu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar
/home/ubuntu/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar
/home/ubuntu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar
/home/ubuntu/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar
/home/ubuntu/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar
/home/ubuntu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar
/home/ubuntu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar
/home/ubuntu/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar
/home/ubuntu/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar
/home/ubuntu/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar
/home/ubuntu/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar
/home/ubuntu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar
/home/ubuntu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar
/home/ubuntu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar
/home/ubuntu/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar
/home/ubuntu/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar
/home/ubuntu/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar
/home/ubuntu/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar
/home/ubuntu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar
/home/ubuntu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar
/home/ubuntu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar
/home/ubuntu/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar
/home/ubuntu/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-auth/2.7.1/hadoop-auth-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar
/home/ubuntu/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar
/home/ubuntu/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar
/home/ubuntu/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar
/home/ubuntu/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar
/home/ubuntu/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar
/home/ubuntu/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar
/home/ubuntu/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar
/home/ubuntu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar
/home/ubuntu/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar
/home/ubuntu/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.7.1/hadoop-hdfs-2.7.1.jar
/home/ubuntu/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar
/home/ubuntu/.m2/repository/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar
/home/ubuntu/.m2/repository/xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar
/home/ubuntu/.m2/repository/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar
/home/ubuntu/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-app/2.7.1/hadoop-mapreduce-client-app-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/2.7.1/hadoop-mapreduce-client-common-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-client/2.7.1/hadoop-yarn-client-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.7.1/hadoop-mapreduce-client-shuffle-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-api/2.7.1/hadoop-yarn-api-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.7.1/hadoop-mapreduce-client-core-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-common/2.7.1/hadoop-yarn-common-2.7.1.jar
/home/ubuntu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar
/home/ubuntu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar
/home/ubuntu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar
/home/ubuntu/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar
/home/ubuntu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar
/home/ubuntu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.7.1/hadoop-mapreduce-client-jobclient-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-annotations/2.7.1/hadoop-annotations-2.7.1.jar
/home/ubuntu/.m2/repository/junit/junit/3.8.1/junit-3.8.1.jar
/home/ubuntu/.m2/repository/org/apache/mrunit/mrunit/1.1.0/mrunit-1.1.0-hadoop2.jar
/home/ubuntu/.m2/repository/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
/home/ubuntu/.m2/repository/org/mockito/mockito-core/1.9.5/mockito-core-1.9.5.jar
/home/ubuntu/.m2/repository/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar
/home/ubuntu/.m2/repository/org/objenesis/objenesis/1.0/objenesis-1.0.jar
/home/ubuntu/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar
/home/ubuntu/.m2/repository/org/powermock/powermock-core/1.5.1/powermock-core-1.5.1.jar
/home/ubuntu/.m2/repository/org/powermock/powermock-reflect/1.5.1/powermock-reflect-1.5.1.jar
/home/ubuntu/.m2/repository/org/javassist/javassist/3.18.0-GA/javassist-3.18.0-GA.jar
/home/ubuntu/.m2/repository/org/powermock/powermock-api-mockito/1.5.1/powermock-api-mockito-1.5.1.jar
/home/ubuntu/.m2/repository/org/powermock/powermock-api-support/1.5.1/powermock-api-support-1.5.1.jar
/home/ubuntu/.m2/repository/org/powermock/powermock-module-junit4/1.5.1/powermock-module-junit4-1.5.1.jar
/home/ubuntu/.m2/repository/org/powermock/powermock-module-junit4-common/1.5.1/powermock-module-junit4-common-1.5.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-minicluster/2.7.1/hadoop-minicluster-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-common/2.7.1/hadoop-common-2.7.1-tests.jar
/home/ubuntu/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar
/home/ubuntu/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar
/home/ubuntu/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar
/home/ubuntu/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar
/home/ubuntu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar
/home/ubuntu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar
/home/ubuntu/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar
/home/ubuntu/.m2/repository/asm/asm/3.1/asm-3.1.jar
/home/ubuntu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar
/home/ubuntu/.m2/repository/org/apache/httpcomponents/httpcore/4.1.2/httpcore-4.1.2.jar
/home/ubuntu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar
/home/ubuntu/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.7.1/hadoop-hdfs-2.7.1-tests.jar
/home/ubuntu/.m2/repository/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-tests/2.7.1/hadoop-yarn-server-tests-2.7.1-tests.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-common/2.7.1/hadoop-yarn-server-common-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.7.1/hadoop-yarn-server-nodemanager-2.7.1.jar
/home/ubuntu/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar
/home/ubuntu/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar
/home/ubuntu/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar
/home/ubuntu/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.7.1/hadoop-yarn-server-resourcemanager-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.7.1/hadoop-yarn-server-applicationhistoryservice-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.7.1/hadoop-yarn-server-web-proxy-2.7.1.jar
/home/ubuntu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6-tests.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.7.1/hadoop-mapreduce-client-jobclient-2.7.1-tests.jar
/home/ubuntu/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar
/home/ubuntu/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar
/home/ubuntu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-hs/2.7.1/hadoop-mapreduce-client-hs-2.7.1.jar

提前致谢!

0 个答案:

没有答案