Question

错误消息：

java.io.IOException: Type mismatch in value from map: expected org.apache.hadoop.io.Text, recieved org.apache.hadoop.io.LongWritable
        at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:918)
        at org.apache.hadoop.mapred.MapTask$OldOutputCollector.collect(MapTask.java:512)
        at main.java.countdestinfo.myTask$Map.map(myTask.java:66)
        at main.java.countdestinfo.myTask$Map.map(myTask.java:45)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:374)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:324)
        at org.apache.hadoop.mapred.Child.main(Child.java:165)

我的代码（这里只有地图代码）：

 44 public class myTask{
 45     public static class Map extends MapReduceBase implements Mapper<BytesWritable, BytesWritable, Text, LongWritable>{
 47         public void map(BytesWritable key, BytesWritable value,
 48                 OutputCollector<Text, LongWritable> output, Reporter reporter) throws IOException{
 66             output.collect(new Text("xxxxxxxxxxxxxxxxx" + "xxx"), new LongWritable(1));
 88         }
 89     }

此处省略的所有行都是注释。

我通过了Text，但声称已收到LongWritable ..

部首：

  3 import java.util.ArrayList;
  4 import java.util.HashMap;
  5 import java.util.List;
  6 import java.util.*;
  7 import java.lang.*;
  8 import java.io.IOException;
  9 import java.util.Iterator;
 10 
 11 import org.apache.hadoop.conf.Configured;
 12 import org.apache.hadoop.fs.FileSystem;
 13 import org.apache.hadoop.fs.Path;
 14 import org.apache.hadoop.io.SequenceFile;
 15 import org.apache.hadoop.io.Text;
 16 import org.apache.hadoop.io.*;
 17 import org.apache.hadoop.mapred.JobClient;
 18 import org.apache.hadoop.mapred.JobConf;
 19 import org.apache.hadoop.mapred.MapReduceBase;
 20 import org.apache.hadoop.mapred.Mapper;
 21 import org.apache.hadoop.mapred.OutputCollector;
 22 import org.apache.hadoop.mapred.Reducer;
 23 import org.apache.hadoop.mapred.Reporter;
 24 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 25 import org.apache.hadoop.util.GenericOptionsParser;
 26 import org.apache.hadoop.util.Tool;
 27 import org.apache.hadoop.mapred.FileInputFormat;
 28 import org.apache.hadoop.mapred.FileOutputFormat;
 29 import org.apache.hadoop.mapred.TextInputFormat;
 30 import org.apache.hadoop.mapred.TextOutputFormat;
 31 
 32 
 33 import java.io.InputStreamReader;
 34 import java.util.HashSet;
 35 import java.util.Iterator;
 37 import org.apache.hadoop.io.LongWritable;
 38 import org.apache.hadoop.io.BytesWritable;
 39 
 40 import com.google.protobuf.InvalidProtocolBufferException;
 41 import display.algo.logs.proto.MiddataMessage;
 42 import java.util.Arrays;

减少和主要功能：

 90     public static class Reduce extends MapReduceBase implements Reducer<Text, LongWritable, Text, Text>{
 91         public void reduce(Text key, Iterator<Text>values,
 92                 OutputCollector<Text, LongWritable> output, Reporter reporter) throws IOException{
 93             long sum = 0;
 94             while(values.hasNext()){
 95                 sum += 1;
 96             }
 97             output.collect(key,new Text(String.valueOf(sum)));
 98         }
 99     }
100 
101     public static void main(String[] args) throws Exception {
102         JobConf conf = new JobConf(myTask.class);
103         conf.setJobName("myTask");           
104 
105         conf.setOutputKeyClass(Text.class);    
106         conf.setOutputValueClass(Text.class);   
107 
108         conf.setMapperClass(Map.class);         
109         conf.setCombinerClass(Reduce.class);         
110         conf.setReducerClass(Reduce.class);        
111 
112         conf.setInputFormat(SequenceFileInputFormat.class);    
113         conf.setOutputFormat(TextOutputFormat.class);  
114         conf.setStrings("mapred.reduce.tasks", "500");
115 
120         FileInputFormat.addInputPath(conf, new Path(args[0]));
121         FileOutputFormat.setOutputPath(conf, new Path(args[1]));
122         JobClient.runJob(conf);
123 
124     }

Answer 1

reduce的输入值应为LongWritable

public void reduce(Text key, Iterator<Text>values,
 OutputCollector<Text, LongWritable> output, Reporter reporter)

应该是：

public void reduce(Text key, Iterator<LongWritable>values,
 OutputCollector<Text, Text> output, Reporter reporter)

hadoop工作失败了，奇怪的例外

1 个答案: