Question

我的hadoop程序如下所示。我把相关代码的片段。我将在main中读取BiG_DATA的参数传递为true。主要是“打印大数据”。但是当涉及RowPreMap类中的map方法时，BIG_DATA的值是其初始化值false。不知道为什么会这样。我错过了什么吗？当我在一台独立的机器上运行它时，这是有效的，但是当我在hadoop集群上执行此操作时则不行。作业由JobControl处理。它是线程的东西吗？

公共类UVDriver扩展了Configured implements Tool {

    public static class RowMPreMap extends MapReduceBase implements
            Mapper<LongWritable, Text, Text, Text> {

        private Text keyText = new Text();
        private Text valText = new Text();

        public void map(LongWritable key, Text value,
                OutputCollector<Text, Text> output, Reporter reporter)
                throws IOException {

            // Input: (lineNo, lineContent)

            // Split each line using seperator based on the dataset.
            String line[] = null;
            if (Settings.BIG_DATA)
                line = value.toString().split("::");
            else
                line = value.toString().split("\\s");

            keyText.set(line[0]);
            valText.set(line[1] + "," + line[2]);

            // Output: (userid, "movieid,rating")
            output.collect(keyText, valText);

        }
    }

    public static class Settings {

        public static boolean BIG_DATA = false;

        public static int noOfUsers = 0;
        public static int noOfMovies = 0;

        public static final int noOfCommonFeatures = 10;
        public static final int noOfIterationsRequired = 3;
        public static final float INITIAL_VALUE = 0.1f;

        public static final String NORMALIZE_DATA_PATH_TEMP = "normalize_temp";
        public static final String NORMALIZE_DATA_PATH = "normalize";
        public static String INPUT_PATH = "input";
        public static String OUTPUT_PATH = "output";
        public static String TEMP_PATH = "temp";

    }

    public static class Constants {

        public static final int BIG_DATA_USERS = 71567;
        public static final int BIG_DATA_MOVIES = 10681;
        public static final int SMALL_DATA_USERS = 943;
        public static final int SMALL_DATA_MOVIES = 1682;

        public static final int M_Matrix = 1;
        public static final int U_Matrix = 2;
        public static final int V_Matrix = 3;
    }

    public int run(String[] args) throws Exception {

        // 1. Pre-process the data.
        // a) Normalize
        // 2. Initialize the U, V Matrices
        // a) Initialize U Matrix
        // b) Initialize V Matrix
        // 3. Iterate to update U and V.

        // Write Job details for each of the above steps.

        Settings.INPUT_PATH = args[0];
        Settings.OUTPUT_PATH = args[1];
        Settings.TEMP_PATH = args[2];
        Settings.BIG_DATA = Boolean.parseBoolean(args[3]);

        if (Settings.BIG_DATA) {
            System.out.println("Working on BIG DATA.");
            Settings.noOfUsers = Constants.BIG_DATA_USERS;
            Settings.noOfMovies = Constants.BIG_DATA_MOVIES;
        } else {
            System.out.println("Working on Small DATA.");
            Settings.noOfUsers = Constants.SMALL_DATA_USERS;
            Settings.noOfMovies = Constants.SMALL_DATA_MOVIES;
        }

            // some code here

            handleRun(control);


        return 0;
    }

    public static void main(String args[]) throws Exception {

        System.out.println("Program started");
        if (args.length != 4) {
            System.err
                    .println("Usage: UVDriver <input path> <output path> <fs path>");
            System.exit(-1);
        }

        Configuration configuration = new Configuration();
        String[] otherArgs = new GenericOptionsParser(configuration, args)
                .getRemainingArgs();
        ToolRunner.run(new UVDriver(), otherArgs);
        System.out.println("Program complete.");
        System.exit(0);
    }

}

工作控制。

public static class JobRunner implements Runnable {
        private JobControl control;

        public JobRunner(JobControl _control) {
            this.control = _control;
        }

        public void run() {
            this.control.run();
        }
    }

    public static void handleRun(JobControl control)
            throws InterruptedException {
        JobRunner runner = new JobRunner(control);
        Thread t = new Thread(runner);
        t.start();

        int i = 0;
        while (!control.allFinished()) {
            if (i % 20 == 0) {
                System.out
                        .println(new Date().toString() + ": Still running...");
                System.out.println("Running jobs: "
                        + control.getRunningJobs().toString());
                System.out.println("Waiting jobs: "
                        + control.getWaitingJobs().toString());
                System.out.println("Successful jobs: "
                        + control.getSuccessfulJobs().toString());
            }
            Thread.sleep(1000);
            i++;
        }

        if (control.getFailedJobs() != null) {
            System.out.println("Failed jobs: "
                    + control.getFailedJobs().toString());
        }
    }

Answer 1

这不起作用，因为静态修饰符的范围不跨越JVM的多个实例（更不用说网络了。）

映射任务始终在单独的JVM中运行，即使它在工具运行器的本地运行。 mapper类仅使用类名实例化，无法访问您在工具运行器中设置的信息。

这是配置框架存在的一个原因。

在Hadoop集群中使用另一个类的静态变量

1 个答案: