Question

我正面对NullPointerException以下代码。如果有人能够审查并帮助我完成该计划，那将是很棒的。

映射器运行正常，但是当我尝试在迭代器中拆分值时，我得到了一个NPE。请帮我弄清楚我的错误。我已经将映射器附在下面了。

Toppermain.java

package TopperPackage;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;


public class TopperMain {
//hadoop jar worcount.jar ars[0] args[1]
public static void main(String[] args) throws Exception {
Job myhadoopJob = new Job();

myhadoopJob.setJarByClass(TopperMain.class);
myhadoopJob.setJobName("Finding topper based on subject");

FileInputFormat.addInputPath(myhadoopJob, new Path(args[0]));
FileOutputFormat.setOutputPath(myhadoopJob, new Path(args[1]));

myhadoopJob.setInputFormatClass(TextInputFormat.class);
myhadoopJob.setOutputFormatClass(TextOutputFormat.class);

myhadoopJob.setMapperClass(TopperMapper.class);
myhadoopJob.setReducerClass(TopperReduce.class);


myhadoopJob.setMapOutputKeyClass(Text.class);
myhadoopJob.setMapOutputValueClass(Text.class);
myhadoopJob.setOutputKeyClass(Text.class);
myhadoopJob.setOutputValueClass(Text.class);

    System.exit(myhadoopJob.waitForCompletion(true) ? 0 : 1);

}

}

TopperMapper.java

package TopperPackage;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/*Surender,87,60,50,50,80
Raj,80,70,80,85,60
Anten,81,60,50,70,100
Dinesh,60,90,80,80,70
Priya,80,85,91,60,75
*/
public class TopperMapper extends Mapper<LongWritable, Text, Text, Text>

{
String temp,temp2;

protected void map(LongWritable key, Text value,Context context)
    throws IOException, InterruptedException {
String record = value.toString();
String[] parts = record.split(",");
temp=parts[0];

temp2=temp+ "\t" + parts[1];
context.write(new Text("Tamil"),new Text(temp2));
temp2=temp+ "\t" + parts[2];
context.write(new Text("English"),new Text(temp2));

temp2=temp+ "\t" + parts[3];
context.write(new Text("Maths"),new Text(temp2));

temp2=temp+ "\t" + parts[4];
context.write(new Text("Science"),new Text(temp2));

temp2=temp+ "\t" + parts[5];
context.write(new Text("SocialScrience"),new Text(temp2));


       }
}

TopperReduce.java

package TopperPackage;

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;



public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names;
private int[] marks;
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
marks = null;
String befsplit;
String[] parts=null;
names = null;
for (Text t : values)
{
befsplit= t.toString();
parts = befsplit.split("\t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;

}
topmark=calcTopper(marks);
top=names[topmark]+ "\t"+marks[topmark]  ;
context.write(new Text(key), new Text(top));
}
public int calcTopper(int[] marks)
{
int count=marks.length;
temp=((marks[1]));
int i=0;
for (i=1;i<=(count-2);i++)
{
   if(temp < marks[i+1])
   {
   temp = marks[i+1];

   }

}
return i;
}
}

错误是

cloudera@cloudera-vm:~/Jarfiles$ hadoop jar TopperMain.jar /user/cloudera/inputfiles/topper/topperinput.txt /user/cloudera/outputfiles/topper/
14/08/24 23:17:07 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
14/08/24 23:17:08 INFO input.FileInputFormat: Total input paths to process : 1
14/08/24 23:17:09 INFO mapred.JobClient: Running job: job_201408241907_0012
14/08/24 23:17:10 INFO mapred.JobClient:  map 0% reduce 0%
14/08/24 23:17:49 INFO mapred.JobClient:  map 100% reduce 0%
14/08/24 23:18:03 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_0, Status : FAILED
java.lang.NullPointerException
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
    at org.apache.hadoop.mapred.Child.main(Child.java:262)

attempt_201408241907_0012_r_000000_0: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_0: log4j:WARN Please initialize the log4j system properly.
14/08/24 23:18:22 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_1, Status : FAILED
java.lang.NullPointerException
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
    at org.apache.hadoop.mapred.Child.main(Child.java:262)

attempt_201408241907_0012_r_000000_1: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_1: log4j:WARN Please initialize the log4j system properly.

我从mapper获得了预期的输出，但是当分割输出并存储在变量中时，reducer会抛出错误。

映射器输出

Tamil   Surender    87
English Surender    60
Maths   Surender    50
Science Surender    50
SocialScrience  Surender    80
Tamil   Raj 80
English Raj 70
Maths   Raj 80
Science Raj 85
SocialScrience  Raj 60
Tamil   Anten   81
English Anten   60
Maths   Anten   50
Science Anten   70
SocialScrience  Anten   100
Tamil   Dinesh  60
English Dinesh  90
Maths   Dinesh  80
Science Dinesh  80
SocialScrience  Dinesh  70
Tamil   Priya   80
English Priya   85
Maths   Priya   91
Science Priya   60
SocialScrience  Priya   75

任何指出我的错误的建议都表示赞赏。

Answer 1

错误是由于您将标记和名称数组初始化为 null 而未正确初始化它们。请使用以下减速机类。

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class TopperReduce extends Reducer<Text, Text, Text, Text> {
    int temp;
    private String[] names = new String[10];
    private int[] marks = new int[10];

    public void reduce(Text key, Iterable<Text> values, Context context)
            throws IOException, InterruptedException {
        String top = "";
        int count = 0, topmark;
        String befsplit;
        String[] parts = null;
        for (Text t : values) {
           befsplit = t.toString();
           parts = befsplit.split("\t");
           names[count] = parts[0];
           marks[count] = Integer.parseInt(parts[1]);
           count++;
        }
       topmark = calcTopper(marks);
       top = names[topmark] + "\t" + marks[topmark];
        context.write(new Text(key), new Text(top));
   }

public int calcTopper(int[] marks) {
    int count = marks.length;
    int i = 0;
    int highestMArk = 0;
    int mark = 0;
    int highestMarkIndex = 0;
    for (; i < count; i++) {
        mark = marks[i];
        if (mark > highestMArk) {
            highestMarkIndex = i;

        }
    }
    return highestMarkIndex;
}
}

Answer 2

您指的是一个空数组变量部分，因此您收到此错误，改变你的代码，如下所述，它可以工作

 public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names=new String[20];
private int[] marks= new int[20];
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
for (Text t : values)
{
String befsplit= t.toString();
String[] parts = befsplit.split("\t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;

}
topmark=calcTopper(marks);
top=names[topmark]+ "\t"+marks[topmark]  ;
context.write(new Text(key), new Text(top));
}

hadoop reducer中的空指针异常

2 个答案: