将多个float参数发送到reducer结果空指针异常

Question

我是hadoop的新手。我试图在以下code.mapper中向reducer发送2个浮点参数成功地将参数传递给reducer但是如果我开始运行reducer null指针异常抛出..可以任何人请帮助我。提前谢谢。

public class MaxTemperature extends Configured implements Tool {


public static class MapMapper extends Mapper<LongWritable, Text, Text, PairWritable>{


    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{
        String regex = ",";//''single quote not applicable for comma.
        String[] val = value.toString().split(regex);

        FloatWritable[]  vv = new FloatWritable[2];
        vv[0]= new FloatWritable(Float.parseFloat(val[3]));
        vv[1]=new FloatWritable(Float.parseFloat(val[13]));

        float dd=Float.parseFloat(val[3]);
        PairWritable ddd = new PairWritable();
        context.write(new Text(val[2]), ddd.set(vv[0], vv[1]));

    }

} 


 public static class PairWritable extends ArrayWritable implements Writable{


    public PairWritable() {
        super(FloatWritable.class);
        // TODO Auto-generated constructor stub
    }

    private FloatWritable floatone;
     private FloatWritable floattwo;

     public String toString() {

        String s = Float.toString(floatone.get());
        String  a=Float.toString(floattwo.get());
        String q = s+'\t'+a;  
        return q;
      }



    public void set(float f1, float f2){
         FloatWritable ff1 = new FloatWritable(f1);
         FloatWritable ff2 = new FloatWritable(f1);
         set(ff1, ff2);
     }

     public PairWritable set(FloatWritable f1, FloatWritable f2){
         this.floatone=f1;
         this.floattwo=f2;
         return this;
     }

     public float getone(){
         return floatone.get();
     }

     public float gettwo(){
         return floattwo.get();
     }


    public void write(DataOutput out) throws IOException {
        // TODO Auto-generated method stub
        this.floatone.write(out);
        this.floattwo.write(out);


    }

    public void readFields(DataInput in) throws IOException {
        // TODO Auto-generated method stub
        this.floatone.readFields(in);
        this.floattwo.readFields(in);

    }


 }




public static class Mapreducers extends Reducer<Text,PairWritable, Text,PairWritable>{

    public void reduce(Text key, Iterable<PairWritable> values,Context context) throws IOException, InterruptedException{


        float sumone =0;
        float sumtwo=0;

        for(PairWritable dd: values){
        sumone+=dd.getone();
        sumtwo+=dd.gettwo();

        }
        FloatWritable result1 = new FloatWritable(sumone);
        FloatWritable result2 = new FloatWritable(sumtwo);
        PairWritable ddd = new PairWritable();
        context.write(key, ddd.set(result1, result2));


}

}


public int run(String[] args) throws Exception {
    Job job = new Job();
    job.setJarByClass(MaxTemperature.class);
    job.setJobName("MaxTemperature");

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(args[0]), conf);

    if(fs.exists(new Path(args[1]))){
        fs.delete(new Path(args[1]),true);
    }

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(MapMapper.class);
    //job.setCombinerClass(Mapreducers.class);
    //job.setNumReduceTasks(0);
    job.setReducerClass(Mapreducers.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(PairWritable.class);

    return job.waitForCompletion(true)?0:1;
}







public static void main(String[] args) throws Exception{
    int xx =1;
    xx = ToolRunner.run(new MaxTemperature(), args);
    System.exit(xx);    
}



}

Answer 1

我怀疑问题只是在反序列化时，没有任何内容填充floatone和floattwo字段。您正在尝试在不存在的对象中填充数据。这样：

public void readFields(DataInput in) throws IOException {
    this.floatone.readFields(in);
    this.floattwo.readFields(in);
}

应该是：

public void readFields(DataInput in) throws IOException {
    floatone = new FloatWritable();  
    floattwo = new FloatWritable(); 
    floatone.readFields(in);
    floattwo.readFields(in);
}

或者，同时更改write和readFields：

public void write(DataOutput out) throws IOException {
    out.writeFloat(floatone.get());
    out.writeFloat(floattwo.get());
}

public void readFields(DataInput in) throws IOException {
    floatone = new FloatWritable(in.readFloat());
    floattwo = new FloatWritable(in.readFloat());
}

这看起来更清晰，对我来说可能更有效率。