Hadoop和自定义可写问题

时间:2016-03-22 21:11:24

标签: hadoop

我使用的是Hadoop 2.7,使用自定义Writable“TextPair”时遇到了问题(“权威指南”第104页)。基本上,当我使用Text时我的程序工作正常,而在使用TextPair时它输出“test.TextTuple@3b86249a test.TextTuple@63cd18fd”。

请知道我的代码有什么问题(如下)?

============

Mapper1:

public class KWMapper extends Mapper<LongWritable, Text, TextTuple, TextTuple> {

    @Override
    public void map(LongWritable k, Text v, Mapper.Context c) throws IOException, InterruptedException {
        String keywordRelRecord[] = v.toString().split(",");
        String subTopicID  = keywordRelRecord[0];
        String paperID    = keywordRelRecord[1];

        //set the KEY
        TextTuple key = new TextTuple();
        key.setNaturalKey(new Text(subTopicID));
        key.setSecondaryKey(new Text("K"));

        //set the VALUE
        TextTuple value = new TextTuple();
        value.setNaturalKey(new Text(paperID));
        value.setSecondaryKey(new Text("K"));

        c.write(key, value);
    }

Mapper2:

public class TDMapper extends Mapper<LongWritable, Text, TextTuple, TextTuple> {

    @Override
    public void map(LongWritable k, Text v, Mapper.Context c) throws IOException, InterruptedException {
        String topicRecord[] = v.toString().split(",");
        String superTopicID = topicRecord[0];
        String subTopicID = topicRecord[1].substring(1, topicRecord[1].length() - 1);

        TextTuple key = new TextTuple();
        key.setNaturalKey(new Text(subTopicID));
        key.setSecondaryKey(new Text("T"));

        TextTuple value = new TextTuple();
        value.setNaturalKey(new Text(superTopicID));
        value.setSecondaryKey(new Text("T"));

        c.write(key, value);
    }

REDUCER:

public class TDKRReducer extends Reducer<TextTuple, TextTuple, Text, Text>{

    public void reduce(TextTuple k, Iterable<TextTuple> values, Reducer.Context c) throws IOException, InterruptedException{                 

        for (TextTuple val : values) {
            c.write(k.getNaturalKey(), val.getNaturalKey());
        }
    }

}

DRIVER:

public class TDDriver {

    public static void main(String args[]) throws IOException, InterruptedException, ClassNotFoundException {

        // This class support the user for the  configuration of the execution;
        Configuration confStage1 = new Configuration();

        Job job1 = new Job(confStage1, "TopDecKeywordRel");
        // Setting the driver class
        job1.setJarByClass(TDDriver.class);


        // Setting the input Files and processing them using the corresponding mapper class
        MultipleInputs.addInputPath(job1, new Path(args[0]), TextInputFormat.class, TDMapper.class);
        MultipleInputs.addInputPath(job1, new Path(args[1]), TextInputFormat.class, KWMapper.class);

        job1.setMapOutputKeyClass(TextTuple.class);
        job1.setMapOutputValueClass(TextTuple.class);

        // Setting the Reducer Class;
        job1.setReducerClass(TDKRReducer.class);
        // Setting the output class for the Key-value pairs

        job1.setOutputKeyClass(Text.class);
        job1.setOutputValueClass(Text.class);
        // Setting the output file 
        Path outputPA = new Path(args[2]);
        FileOutputFormat.setOutputPath(job1, outputPA);

        // Submitting the Job Monitoring the execution of the Job
        System.exit(job1.waitForCompletion(true) ? 0 : 1);
        //conf.setPartitionerClass(CustomPartitioner.class);    
    }

}

CUSTOM VARIABLE

public class TextTuple implements Writable, WritableComparable<TextTuple> {

    private Text naturalKey;
    private Text secondaryKey;

    public TextTuple() {
        this.naturalKey = new Text();
        this.secondaryKey = new Text();
    }


    public void setNaturalKey(Text naturalKey) {
        this.naturalKey = naturalKey;
    }

    public void setSecondaryKey(Text secondaryKey) {
        this.secondaryKey = secondaryKey;
    }

    public Text getNaturalKey() {
        return naturalKey;
    }

    public Text getSecondaryKey() {
        return secondaryKey;
    }


    @Override
    public void write(DataOutput out) throws IOException {
        naturalKey.write(out);
        secondaryKey.write(out);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        naturalKey.readFields(in);
        secondaryKey.readFields(in);
    }

    //This comparator controls the sort order of the keys.
    @Override   
    public int compareTo(TextTuple o) {
        // comparing the naturalKey
        int compareValue = this.naturalKey.compareTo(o.naturalKey);
        if (compareValue == 0) {
            compareValue = this.secondaryKey.compareTo(o.secondaryKey);
        }
        return -1 * compareValue; 
    }

}

0 个答案:

没有答案