Question

我在学校集群中运行hadoop。我在线程main中遇到异常，找不到类。

Exception in thread "main" java.lang.ClassNotFoundException: movielens.MovieLensDriver
    at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:423)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:264)
    at org.apache.hadoop.util.RunJar.main(RunJar.java:153)

但是我知道我必须在命令中使用完整的包名，我也做了同样的事情。以下是我使用的命令

hadoop jar movielens.jar movielens.MovieLensDriver input output

以下是我的驱动程序类的代码。

package movielens;

import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;

public class MovieLensDriver {

    public static class JobRunner implements Runnable {
        private JobControl control;

        public JobRunner(JobControl _control) {
            this.control = _control;
        }

        public void run() {
            this.control.run();
        }
    }

    public static void handleRun(JobControl control)
            throws InterruptedException {
        JobRunner runner = new JobRunner(control);
        Thread t = new Thread(runner);
        t.start();

        while (!control.allFinished()) {
            System.out.println("Still running...");
            Thread.sleep(5000);
        }
    }

    public static void main(String args[]) throws IOException,
            InterruptedException {

        System.out.println("Program started");
        if (args.length != 2) {
            System.err
                    .println("Usage: MovieLensDriver <input path> <output path>");
            System.exit(-1);
        }

        JobConf conf1 = new JobConf(movielens.MovieLensDriver.class);
        conf1.setMapperClass(MoviePairsMapper.class);
        conf1.setReducerClass(MoviePairsReducer.class);

        conf1.setJarByClass(MovieLensDriver.class);

        FileInputFormat.addInputPath(conf1, new Path(args[0]));
        FileOutputFormat.setOutputPath(conf1, new Path("temp"));

        conf1.setMapOutputKeyClass(Text.class);
        conf1.setMapOutputValueClass(Text.class);

        conf1.setOutputKeyClass(Text.class);
        conf1.setOutputValueClass(IntWritable.class);

        JobConf conf2 = new JobConf(MovieLensDriver.class);
        conf2.setMapperClass(MoviePairsCoOccurMapper.class);
        conf2.setReducerClass(MoviePairsCoOccurReducer.class);

        conf2.setJarByClass(MovieLensDriver.class);

        FileInputFormat.addInputPath(conf2, new Path("temp"));
        FileOutputFormat.setOutputPath(conf2, new Path(args[1]));

        conf2.setInputFormat(KeyValueTextInputFormat.class);

        conf2.setMapOutputKeyClass(Text.class);
        conf2.setMapOutputValueClass(IntWritable.class);

        conf2.setOutputKeyClass(Text.class);
        conf2.setOutputValueClass(IntWritable.class);

        Job job1 = new Job(conf1);
        Job job2 = new Job(conf2);

        JobControl jobControl = new JobControl("jobControl");
        jobControl.addJob(job1);
        jobControl.addJob(job2);
        job2.addDependingJob(job1);
        handleRun(jobControl);

        System.out.println("Program complete.");

        System.exit(0);
    }
}

过去3个小时对这个错误进行了令人沮丧的搜索，并且感谢任何帮助。

Answer 1

您可以尝试'libjar'选项，它将获取jar并将其放在分布式缓存中。这使得jar可用于所有作业的任务尝试。请注意，libjars参数采用逗号分隔列表，而不是冒号或分号分隔列表。

export LIBJARS=/path/jars1,/path/jars2,/path/movielens.jar

hadoop jar movielens.jar movielens.MovieLensDriver -libjars ${LIBJARS} input output

Thread Main中的异常：ClassNotFoundException

1 个答案: