Question

嘿伙计们这可能听起来有点幼稚的问题，但我是mapreduce的新手

我正在实现mapreduce作业，我在地图中有一些sysout语句，只是为了查看地图中的内容并减少边数，但在完成作业后job.waitForCompletion()返回false并且作业失败

我试图调试它，但没有发现任何可疑的东西，所以在这里发布代码

CustomKey.java

package com.example.secondarysort;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;

public class CustomKey {
    Text key = new Text();
    IntWritable value = new IntWritable();

    public void set(Text key,IntWritable value){
        this.key = key;
        this.value = value;
    }

    public Text getKey(){
        return this.key;
    }

    public IntWritable getValue(){
        return this.value;
    }
}



SSDriver.java

package com.example.secondarysort;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;


public class SSDriver {

    private static class SSMapper extends Mapper<LongWritable, Text, CustomKey,NullWritable>{

        CustomKey customKey = new CustomKey();
        @Override
        public void map(LongWritable key, Text value,Context context)
                throws IOException, InterruptedException {
            System.out.println(value);
            StringTokenizer tokens = new StringTokenizer(value.toString());
            customKey.set(new Text(tokens.nextToken()), new IntWritable(Integer.parseInt(tokens.nextToken())));
            context.write(customKey, NullWritable.get());
        }

    }

    private static class SSReducer extends Reducer<CustomKey, NullWritable, Text, IntWritable>{

        @Override
        public void reduce(CustomKey key, Iterable<NullWritable> values,Context context)
                throws IOException, InterruptedException {
            System.out.println(key.getKey()+" "+key.getValue());
            context.write(key.getKey(), key.getValue());
        }
    }

    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "secondary_sort");

        job.setJarByClass(SSDriver.class);

        job.setMapperClass(SSMapper.class);
        job.setReducerClass(SSReducer.class);

        job.setMapOutputKeyClass(CustomKey.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        TextInputFormat.addInputPath(job, new Path("data/data.txt"));
        TextOutputFormat.setOutputPath(job, new Path("data/output"));

        try {
            System.out.println(job.waitForCompletion(true));
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

Answer 1

问题是您使用的是未实现CustomKey界面的自定义密钥WritableComparable 您的自定义密钥必须实现它，并提供两个方法的定义，例如write()和readFields()。重要的是覆盖hashCode()。您可以参考以获取一段代码供您参考here
希望它有所帮助！

mapreduce作业未使用LocalJobRunner运行

1 个答案: