Question

我正在尝试使用avro运行map reduce但是在尝试了网络上人们建议的所有选项之后，我仍然无法通过 -

15/11/09 21:54:48 WARN mapred.LocalJobRunner：job_local1421922570_0001 java.lang.Exception：java.lang.NoSuchMethodError：org.apache.avro.generic.GenericData.createDatumWriter（Lorg / apache / avro / Schema;）Lorg / apache / avro / io / DatumWriter;

以下是代码（来自Hadoop权威指南）公共类AvroGenericMapTemperature扩展了Configured implements Tool {

private static final Schema SCHEMA = new Schema.Parser().parse(
    "{" +
    " \"type\" : \"record\"," +
    " \"name\" : \"WeatherRecord\"," +
    " \"doc\" : \"A weather reading\"," +
    " \"fields\": [" +
    "    {\"name\": \"year\", \"type\": \"int\" }," +
    "    {\"name\": \"temperature\", \"type\": \"int\" }," +
    "    {\"name\": \"stationId\", \"type\": \"string\" }" +
    "  ]" +
    "}"
);

public static class MaxTemperatureMapper extends 
    Mapper<LongWritable,  Text, 
    AvroKey<Integer>,AvroValue<GenericRecord> > {
    private NcdcRecordParser parser = new NcdcRecordParser();
    private GenericRecord record = new GenericData.Record(SCHEMA);

    @Override
    protected void map(
        LongWritable key,
        Text value,
        Mapper<LongWritable, Text, AvroKey<Integer>, 
        AvroValue<GenericRecord>>.Context context)
        throws IOException, InterruptedException {
        parser.parse(value.toString());

        if( parser.isValidTemperature() ) {
            record.put("year", parser.getYearInt());
            record.put("temperature", parser.getAirTemperature());
            record.put("stationId", parser.getStationId());
            context.write(new AvroKey<Integer>(parser.getYearInt()), 
            new AvroValue<GenericRecord>(record));
        }
    }
}

public static class MaxTemperatureReducer extends 
Reducer<AvroKey<Integer>, AvroKey<GenericRecord>, 
AvroKey<GenericRecord>, NullWritable> {

    @Override
    protected void reduce(
        AvroKey<Integer> key,
        Iterable<AvroKey<GenericRecord>> values,
        Reducer<AvroKey<Integer>, AvroKey<GenericRecord>, 
        AvroKey<GenericRecord>, NullWritable>.Context context)
        throws IOException, InterruptedException {
        GenericRecord max = null;

        for ( AvroKey<GenericRecord> value : values) {
            GenericRecord record = value.datum();
            if ( max == null ||
                 (Integer)record.get("temperature") > (Integer) 
                 max.get("termperature") ) {
                 max = newWeatherRecord(record);
            }
        }
        context.write(new AvroKey<GenericRecord>(max), 
        NullWritable.get());
    }

    private GenericRecord newWeatherRecord(GenericRecord value) {
        GenericRecord record = new GenericData.Record(SCHEMA);
        record.put("year", value.get("year"));
        record.put("temperature", value.get("temperature"));
        record.put("stationId", value.get("stationId"));
        return record;
    }
}

public int run(String[] args) throws Exception {
    // TODO Auto-generated method stub
    Job job = new Job( getConf(), "Avro mapreduce");
    job.setJarByClass(getClass());


    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    AvroJob.setMapOutputKeySchema(job, Schema.create(Schema.Type.INT));
    AvroJob.setMapOutputValueSchema(job, SCHEMA);
    AvroJob.setOutputKeySchema(job, SCHEMA);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(AvroKeyOutputFormat.class);

    job.setMapperClass(MaxTemperatureMapper.class);
    job.setReducerClass(MaxTemperatureReducer.class);

    job.waitForCompletion(true);
    return 0;
}

public static void main(String[] args) throws Exception {
    int exitcode = 
        ToolRunner.run(new AvroGenericMapTemperature(), args);
}

}; 我已经在HADOOP_CLASSPATH（avro，avro-mapred，avro-tools等1.7.5版本）中明确设置了所有avro jar，并在运行上面时指定了-D mapreduce.job.user.classpath.first = true，但是我继续得到同样的错误....我知道默认avro与hadoop 2.6.0是1.7.4我甚至尝试了该版本的avro，但没有运气....任何帮助将是巨大的

hadoop 2.6.0和avro

0 个答案: