Question

今天我在两个节点的hadoop上运行了一个作业。我的程序在地图100％之后挂起而且reduce无法运行。返回的消息是：

14/01/15 20:30:49 INFO input.FileInputFormat: Total input paths to process : 25
14/01/15 20:30:49 INFO util.NativeCodeLoader: Loaded the native-hadoop library
14/01/15 20:30:49 WARN snappy.LoadSnappy: Snappy native library not loaded
14/01/15 20:30:51 INFO mapred.JobClient: Running job: job_201401152029_0001
14/01/15 20:30:52 INFO mapred.JobClient:  map 0% reduce 0%
14/01/15 20:30:58 INFO mapred.JobClient:  map 7% reduce 0%
14/01/15 20:31:01 INFO mapred.JobClient:  map 15% reduce 0%
14/01/15 20:31:22 INFO mapred.JobClient:  map 100% reduce 0%

我的源代码是：

PollingImage.java：

package imagePolling;
import java.io.IOException;
import java.util.*;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.conf.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.util.*;

public class PollingImage extends Configured implements Tool{
public int run(String[] args) throws Exception{
    Job job = new Job(getConf());
    job.setJarByClass(PollingImage.class);
    job.setJobName("polling_image");

    job.setMapperClass(PollingMapper.class);
    job.setReducerClass(PollingReducer.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    //job.setNumReduceTasks(0);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    FileInputFormat.setInputPaths(job,new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    boolean success = job.waitForCompletion(true);

    return success?0:1;
}
public static void main(String[] args) {
    try{
        int ret = ToolRunner.run(new PollingImage(), args);
        System.exit(ret);
    }catch(Exception ex){
        System.out.println(ex.getMessage());
    }
}
}

PollingMapper.java：

package imagePolling;
import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;

public class PollingMapper extends Mapper<LongWritable, Text, Text, Text> {
private Text nameKey=new Text();
public void map(LongWritable key,Text value,Context ctx) throws                 IOException,InterruptedException {
    Path filePath=((FileSplit)ctx.getInputSplit()).getPath();
    String name=filePath.getName().replaceAll(".txt", "");
    nameKey = new Text(name);
    ctx.write(nameKey, value);
}
}

PollingReducer.java：

package imagePolling;
import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Reducer;

public class PollingReducer extends Reducer<Text, Text, Text, Text> {
@Override
public void reduce(Text key,Iterable<Text> values,Context ctx) throws IOException,InterruptedException {
    Text v=new Text("a fool");
    ctx.write(key, v);
}
}

我已经阅读了logs.It只显示地图已经完成，而没有关于reduce。

Answer 1

检查你的/ etc / hosts文件并检查IP条目的主机名，如果是伪模式，则提供环回地址。

地图100％后减少无法运行

1 个答案: