Question

我创建了一个文件，并添加了一些数字，如10,20,220和228.我想在我的mapper函数中读取此文件，如下所示，并检查数字是否为Amicable。但是在编译类文件并构建jar后，输出文件中没有任何内容。

public class FriendlyNumbers {

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "befriended numbers");
        job.setJarByClass(FriendlyNumbers.class);
        job.setMapperClass(FriendlyNumberMapper.class);
//      job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(FriendlyNumberKeywordReducer.class);

        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(NumberCouple.class);
        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

class FriendlyNumberMapper extends Mapper<Object, Text, IntWritable, NumberCouple> {

    // process all the input data
    // the data come's from the file file0

    private IntWritable number = new IntWritable(); // number from file
    private IntWritable sum = new IntWritable(); // number from calculateSum()
    private NumberCouple numberCouple = new NumberCouple();

    public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

        StringTokenizer numberTokens = new StringTokenizer(value.toString());

        // loop trough all given numbers
        while (numberTokens.hasMoreTokens()) {

            int parsedNumberToken = Integer.parseInt(numberTokens.nextToken());
            int calculatedSum = calculateSum(parsedNumberToken);

            // set stuff
            number.set(parsedNumberToken);
            sum.set(calculatedSum);
            numberCouple.set(number, sum);

            context.write(sum, numberCouple);

            if (number.get() != sum.get()) {
                context.write(number, numberCouple);
            }
        }
    }

    // the actual sum to check if a number is amicable
    public int calculateSum(int number) {
        int sum = 0;

        for (int i = 1; i <= number / 2; i++) {
            if (number % i == 0) {
                sum += i;
            }
        }
        return sum;
    }
}

class FriendlyNumberKeywordReducer extends Reducer<IntWritable, NumberCouple, IntWritable, IntWritable> {

    // combine data 
    // in this case: get only the befriended numbers and remove others

    public void reduce(IntWritable key, Iterable<NumberCouple> values, Context context) throws IOException, InterruptedException {
        //
    }   
}

class NumberCouple implements WritableComparable<NumberCouple> {

    private IntWritable number;
    private IntWritable sum;

    public NumberCouple() {
        set(new IntWritable(), new IntWritable());
    }

    public NumberCouple(NumberCouple couple) {
        set(new IntWritable(couple.number.get()), new IntWritable(couple.sum.get()));
    }

    public NumberCouple(int number, int sum) {
        set(new IntWritable(number), new IntWritable(sum));
    }

    public void set(IntWritable number, IntWritable sum) {
        this.number = number;
        this.sum = sum;
    }

    public IntWritable getNumber() {
        return this.number;
    }

    public IntWritable getSum() {
        return this.sum;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        number.write(out);
        sum.write(out);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        number.readFields(in);
        sum.readFields(in);
    }

    @Override
    public int compareTo(NumberCouple o) {
         return number.compareTo(o.number);
    }
}

Answer 1

由于您没有将numReduceTask设置为＆＃34; 0＆＃34;所以它将转到Reducer并尝试运行reduce任务。

因此，如果您想要运行仅限地图的作业，请将numReduceTask设置为＆＃34; 0＆＃34;。您不需要设置ReducerClass。在驱动程序类中使用以下。

Job job = Job.getInstance(conf, "befriended numbers");

// Set this property to Zero to run map-only job 
job.setNumReduceTasks(0);

job.setJarByClass(FriendlyNumbers.class);
job.setMapperClass(FriendlyNumberMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(NumberCouple.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);

Hadoop MapReduce不写输出

1 个答案: