我正在CHD5.4上执行MR程序。我正在获取Mapper输出但是reducer部分没有被执行。有人可以帮我这个吗?
import java.io.IOException;
import java.util.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.mapred.MapReduceBase;
import java.util.StringTokenizer;
public class StubDriver extends Configured implements Tool {
// This is main method
public static void main(String[] args) throws Exception {
// new configuration
Configuration conf = new Configuration();
System.exit(ToolRunner.run(conf, new StubDriver(), args));
}
// Run method
public int run (String[] args) throws Exception {
// Job Configuration
Job job = new Job(getConf(), "My Program");
job.setJarByClass(StubDriver.class);
job.setMapperClass(Map1.class);
job.setReducerClass(Reduce1.class);
job.setInputFormatClass (TextInputFormat.class);
job.setOutputFormatClass (TextOutputFormat.class);
job.setOutputKeyClass (IntWritable.class);
job.setOutputValueClass (IntWritable.class);
job.setMapOutputKeyClass (IntWritable.class);
job.setMapOutputValueClass (Text.class);
TextInputFormat.addInputPath(job, new Path (args[0]));
TextOutputFormat.setOutputPath(job, new Path (args[1]));
return job.waitForCompletion(true) ? 0 : 1;
}
public static class Map1 extends Mapper <LongWritable,Text,IntWritable,Text> {
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
StringTokenizer tokenizer = new StringTokenizer(value.toString());
String edge_weight = tokenizer.nextToken();
String source = tokenizer.nextToken();
String destination = tokenizer.nextToken();
context.write(new IntWritable(Integer.parseInt(edge_weight)), new Text(source + ":" + destination));
}
}
public static class Reduce1 extends Reducer <IntWritable,Text,IntWritable,IntWritable> {
public void reducer(IntWritable key, Iterable <Text> values, Context context) throws Exception {
int a = 1;
for (Text val: values) {
context.write(key,new IntWritable(a));
}
}
}
}
Mapper输出:
1 c:f
2 g:i
3 c:e
3 e:f
5 d:h
6 f:h
7 e:d
8 d:g
8 b:d
9 b:c
9 g:h
10 a:b
11 i:h
12 a:c
我也尝试了@Override,但它抛出了错误:方法没有覆盖或实现超类型的方法