在hadoop reducer

Question

当我运行MapReduce作业时，我收到以下错误。

我的职业课程如下：

package mutualfriends;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class MutualSuggest extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {
        // TODO Auto-generated method stub
        if(args.length !=2)
        {
            System.err.println("Usage: MutualSuggest <input path> <outputpath>");
            System.exit(-1);
        }
        Job job = new Job();
        job.setJarByClass(MutualSuggest.class);
        job.setJobName("Mutual Friends");

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        job.setMapperClass(MutualSuggestMapper.class);
        job.setCombinerClass(MutualSuggestReducer.class);
        job.setReducerClass(MutualSuggestReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        System.exit(job.waitForCompletion(true) ? 0:1);
        boolean success = job.waitForCompletion(true);
        return success ? 0 : 1;
    }

    public static void main(String[] args) throws Exception 
    {
        MutualSuggest driver = new MutualSuggest();
        int exitCode = ToolRunner.run(driver, args);
        System.exit(exitCode);
    }

}

我的Mapper类如下所述：

package mutualfriends;

import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

@SuppressWarnings({ "unchecked","rawtypes"})
public class MutualSuggestMapper extends Mapper<Object, Text, Text, Text>{

    public String sorted(String name) {
        char[] chars = name.toCharArray();
        Arrays.sort(chars);
        String sorted = new String(chars);
        return sorted;
    }

    @Override
    public void map(Object key, Text value, Context context)
                throws IOException, InterruptedException {
        String line = value.toString();
        String[] spl = line.split("=");
        String user=spl[0];
        String[] friends = spl[1].split(",");
        Map m = new HashMap();
        for (int i=0;i<friends.length;i++)
        {
            m.put(sorted(user+friends[i]), sorted(spl[1].replace(",","")));
        }
        Set x=m.keySet();
        Iterator ite=x.iterator();
        while (ite.hasNext())
        {
            Object z=ite.next();
            context.write(new Text((String) z),new Text((String) m.get(z)));
        }
    }
}

我的减速机课程如下：

package mutualfriends;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MutualSuggestReducer extends Reducer<Text, Text, Text, Text>{
    @SuppressWarnings({ "rawtypes", "unchecked" })
    @Override
    public void reduce(Text key, Iterable<Text> values,Context context)
            throws IOException, InterruptedException {
        Map hm=new HashMap();
        int z=1;
        for (Text val:values)
        {
            hm.put(z, new Text(val));
            z+=1;
        }

        String s=new String();
        String t=new String();
        s= hm.get(1).toString();
        t= hm.get(2).toString();

        //System.out.println(s+" "+t);


        String x = s.replaceAll("[^" + t + "]", "");

        System.out.println(key+" "+new Text(x));


        context.write(new Text(key),new Text(x));

    }

}

打印时我输入正确，如：

AB CD
AC BD
AD BC
BC ADE
BD ACE
BE CD
CD ABE
CE BD
DE BC

但是在编写输出时如：

context.write(key,new Text(x));

我收到以下错误：

15/07/03 16:13:10 WARN mapred.LocalJobRunner: job_local1502108935_0001
java.lang.NullPointerException
    at mutualfriends.MutualSuggestReducer.reduce(MutualSuggestReducer.java:26)
    at mutualfriends.MutualSuggestReducer.reduce(MutualSuggestReducer.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:177)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:649)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:418)
    at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:398)
15/07/03 16:13:11 INFO mapred.JobClient:  map 100% reduce 0%

如何解决这个问题？

提前致谢。

Answer 1

您正在索引hm.get(1) and hm.get(2)，而不检查地图中是否存在这些关键字。在从HashMap获取值之前进行检查。 非常基本的错误

Answer 2

我通过一个简单的修改来解决这个问题。

删除job.setCombinerClass(MutualSuggestReducer.class);类中的MutualSuggest语句。

出现此错误的原因是，通过调用job.setCombinerClass(MutualSuggestReducer.class);它执行一次Reducer函数，再次通过调用job.setReducerClass(MutualSuggestReducer.class);程序尝试再次执行Reducer函数。所以我删除了job.setCombinerClass(MutualSuggestReducer.class);。和程序完美无缺。