Question

地图阶段运行然后退出而不用打扰减速器。这项工作交替地打印来自mapper的＃Hello;＆＃34;和＃34;写CellWithTotalAmount＆＃34;就是这样。它创建的输出目录为空。

我已经检查了至少十几个其他＆＃34;减速机不会开始＆＃34;问题，但没有找到答案。我已经检查过map的输出与reduce的输入相同，reduce减少使用Iterable，设置了正确的输出类等等。

作业配置

public class HoursJob {
    public static void main(String[] args) throws Exception {
        if (args.length != 2) {
          System.err.println("Usage: HoursJob <input path> <output path>");
          System.exit(-1);
        }

        Job job = Job.getInstance();
        job.setJarByClass(HoursJob.class);
        job.setJobName("Hours job");

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.setMapperClass(HoursMapper.class);
        job.setReducerClass(HoursReducer.class);

        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(CellWithTotalAmount.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        int ret = job.waitForCompletion(true) ? 0 : 1;
        System.exit(ret);
    }
}

映射

public class HoursMapper 
        extends Mapper<LongWritable, Text, IntWritable, CellWithTotalAmount> {
    static double BEGIN_LONG = -74.913585;
    static double BEGIN_LAT = 41.474937;
    static double GRID_LENGTH = 0.011972;
    static double GRID_HEIGHT = 0.008983112;

    @Override
    public void map(LongWritable key, Text value, Mapper.Context context)
            throws IOException, InterruptedException {

        System.out.println("Hello from mapper.");
        String recordString = value.toString();
        try {
            DEBSFullRecord record = new DEBSFullRecord(recordString);
            Date pickupDate = record.getPickup();
            Calendar calendar = GregorianCalendar.getInstance();
            calendar.setTime(pickupDate);
            int pickupHour = calendar.get(Calendar.HOUR_OF_DAY);
            int cellX = (int)
                ((record.getPickupLongitude() - BEGIN_LONG) / GRID_LENGTH) + 1;
            int cellY = (int)
                ((BEGIN_LAT - record.getPickupLatitude()) / GRID_HEIGHT) + 1;

            CellWithTotalAmount hourInfo = 
                new CellWithTotalAmount(cellX, cellY, record.getTotal());
            context.write(new IntWritable(pickupHour), hourInfo);
        } catch (Exception ex) {
            System.out.println(
                "Cannot parse: " + recordString + "due to the " + ex);
        }
    }
}

减速

public class HoursReducer 
        extends Reducer<IntWritable, CellWithTotalAmount, Text, NullWritable> {
    @Override
    public void reduce(IntWritable key, Iterable<CellWithTotalAmount> values, 
            Context context) throws IOException, InterruptedException {
        System.out.println("Hello from reducer.");
        int[][] cellRideCounters = getCellRideCounters(values);
        CellWithRideCount cellWithMostRides = 
            getCellWithMostRides(cellRideCounters);

        int[][] cellTotals = getCellTotals(values);
        CellWithTotalAmount cellWithGreatestTotal = 
            getCellWithGreatestTotal(cellTotals);

        String output = key + " "
            + cellWithMostRides.toString() + " "
            + cellWithGreatestTotal.toString();

        context.write(new Text(output), NullWritable.get());
    }

    //omitted for brevity
}

自定义可写类

public class CellWithTotalAmount implements Writable {
    public int cellX;
    public int cellY;
    public double totalAmount;

    public CellWithTotalAmount(int cellX, int cellY, double totalAmount) {
        this.cellX = cellX;
        this.cellY = cellY;
        this.totalAmount = totalAmount;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        System.out.println("Writing CellWithTotalAmount");
        out.writeInt(cellX);
        out.writeInt(cellY);
        out.writeDouble(totalAmount);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        System.out.println("Reading CellWithTotalAmount");
        cellX = in.readInt();
        cellY = in.readInt();
        totalAmount = in.readDouble();
    }

    @Override
    public String toString() {
        return cellX + " " + cellY + " " + totalAmount;
    }
}

Answer 1

我认为reduce函数存在很多异常，因此Framework无法正常完成工作

        let xhr = new XMLHttpRequest();

        if ('withCredentials' in xhr){
            xhr.open('POST', XHR_URL, true);
        } else if (typeof XDomainRequest != 'undefined'){
            xhr = new XDomainRequest();
            xhr.open('POST', XHR_URL);
        } else {
            xhr = null;
            return;
        }

        xhr.setRequestHeader('Content-type', 'text/plain');

        xhr.onreadystatechange = () => {
            if(xhr.readyState == 4 && xhr.status == 200) {
                // do something
            }
        }

        xhr.onprogress = function () { };
        xhr.ontimeout = function () { };

        setTimeout(function () {
            xhr.send(params);
        }, 0);

添加try-catch以在reduce函数中获取异常
。从catch中的函数返回

。在调用getCellWithMostRiders（..）之前还要添加一个if语句我认为这个问题就在这里。根据你的需要填写if语句我根据我的猜测填写它然后根据你的需要改变它，如果它不适合你

Hadoop MapReduce reducer无法启动

1 个答案: