Question

我正在尝试在我的Reducer中设置Java System属性java.util.Arrays.useLegacyMergeSort，以强制系统使用JDK 6实现Arrays.sort方法，而不是JDK8。

package scoring.devicestatus;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.log4j.Logger;

import domain.DeviceEvent;
import domain.DeviceStatus;
import domain.EndOfPeriodEvent;
import domain.PiidBoundaryEvent;
import util.DateUtils;

public class DeviceStatusReducer extends Reducer<Text, Text, Text, NullWritable> {

    public static final Logger logger = Logger.getLogger(DeviceStatusReducer.class);

    @Override
    public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        -->Configuration conf = context.getConfiguration();
        .       .
        .
        try{
            .
            .
            writeToContext(key, records, context);
        }catch(Throwable t){
            logger.error("Error processing VIN: " + key.toString(), t);
            throw new RuntimeException(t);
        }
    }

    List<DeviceStatus> reduce(Iterable<Text> values, String endDate) {
        List<DeviceEvent> events = createEvents(values, endDate);
        .
        .       
    }

    List<DeviceEvent> createEvents(Iterable<Text> values, String endDate) {
        List<DeviceEvent> events = new ArrayList<DeviceEvent>(10);
        for(Text text : values){
            List<DeviceEvent> instances = DeviceEvent.getInstance(text);
            for(DeviceEvent instance : instances){
                if(!Constants.ORPHAN_PIID.equals(instance.getProgramInstanceId())){
                    events.add(instance);
                }
            }
        }
    --> System.out.println("Reducer Class:"+System.getProperty("java.util.Arrays.useLegacyMergeSort")+" "+conf.get("java.util.Arrays.useLegacyMergeSort"));
        Collections.sort(events);
        return events;
    }

    public void writeToContext(Text key, List<DeviceStatus> deviceStatuses, Context context) throws IOException, InterruptedException {
        String vin = key.toString();
        for (DeviceStatus status : deviceStatuses) {
            context.write(new Text(status.toCsvString(vin)), NullWritable.get());
        }
    }
}

package domain;
    public abstract class DeviceEvent implements Comparable<DeviceEvent> {

        .
        .
        .   
        @Override
        public int compareTo(DeviceEvent arg0) {
            int comparison = getTimeStamp().compareTo(arg0.getTimeStamp());
            if(comparison == 0){
                comparison = isInstalled() ? 1 : -1;
            }
            return comparison;
        }
        .
        .
        .

    }

我使用-D选项作为

通过命令行运行jar

hadoop jar jarname.jar classname -Djava.util.Arrays.useLegacyMergeSort=true args

reducer确实将属性带入了配置，但我的错误仍然没有得到修复。当我使用System.out.println("Reducer Class:" +System.getProperty("java.util.Arrays.useLegacyMergeSort")+ " " +conf.get("java.util.Arrays.useLegacyMergeSort"));

使用configuration.get和System.getProperty打印属性时

看起来像这样：

stdout: Reducer Class:null true Reducer Class:null true Reducer Class:null true Reducer Class:null true Reducer Class:null true Reducer Class:null true

Below is the stack trace: SysLog: 2016-10-05 16:25:21,032 ERROR [main] DeviceStatusReducer: Error processing VIN: 19XXXXXXXXXXX java.lang.IllegalArgumentException: Comparison method violates its general contract! at java.util.ComparableTimSort.mergeLo(ComparableTimSort.java:744) at java.util.ComparableTimSort.mergeAt(ComparableTimSort.java:481) at java.util.ComparableTimSort.mergeCollapse(ComparableTimSort.java:406) at java.util.ComparableTimSort.sort(ComparableTimSort.java:213) at java.util.Arrays.sort(Arrays.java:1312) at java.util.Arrays.sort(Arrays.java:1506) at java.util.ArrayList.sort(ArrayList.java:1454) at java.util.Collections.sort(Collections.java:141) at DeviceStatusReducer.createEvents(DeviceStatusReducer.java:139) at DeviceStatusReducer.reduce(DeviceStatusReducer.java:43) at DeviceStatusReducer.reduce(DeviceStatusReducer.java:34) at DeviceStatusReducer.reduce(DeviceStatusReducer.java:24) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

我确信java.util.Arrays.useLegacyMergeSort可以工作，因为当我在Main类中设置为System.setProperty("java.util.Arrays.useLegacyMergeSort","true")时它在我的eclipse（One JVM）上工作但在Hadoop集群（多个JVM）上不起作用）。我还尝试在Reducer中将其设置为System.setProperty("java.util.Arrays.useLegacyMergeSort","true")。

如何让属性在所有JVM上传播？或者如何将配置属性设置为系统属性？感谢

Answer 1

通过运行jar作为

来工作

hadoop jar JarName.jar MainClassName -D mapreduce.reduce.java.opts=-Djava.util.Arrays.useLegacyMergeSort=true args

请注意，在-D之后有一个空格用于hadoop属性，在-D之后没有空格用于设置JVM属性。

在Reducer中使用System.setProperty（）

1 个答案: