Question

当我尝试使用oozie在mapreduce中运行word-count prg时..它只是读取输入记录并显示它。我猜它甚至没有调用我的mapper和reducer类。因为我正在使用新的API，所以在workflow.xml中也包含了new-api属性标记。

Map-reduce代码段：

public class WordCount {

  public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {
     private final static IntWritable one = new IntWritable(1);
     private Text word = new Text();

     public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
         String line = value.toString();
         StringTokenizer tokenizer = new StringTokenizer(line);
         while (tokenizer.hasMoreTokens()) {
             word.set(tokenizer.nextToken());
             context.write(word, one);
         }
 }

}

  public static class Reduce extends Reducer<Text,IntWritable,Text,IntWritable> {

     public void reduce(Text key, Iterable<IntWritable> values, Context context) 
       throws IOException, InterruptedException {
         int sum = 0;
         for (IntWritable val : values) {
             sum += val.get();
         }
             context.write(key, new IntWritable(sum));
     }

}

我的workflow.xml：

    <?xml version="1.0" encoding="UTF-8"?>
    <workflow-app xmlns='uri:oozie:workflow:0.1' name="wordcount">
    <start to="wc-node" />
    <action name="wc-node">
    <map-reduce>
        <job-tracker>${jobTracker}</job-tracker>
        <name-node>${nameNode}</name-node>
        <prepare>
            <delete path="${nameNode}/user/${wf:user()}/${wordcountRoot}/output-    data/${outputDir}"/>
        </prepare>

        <configuration>

            <property>
                <name>mapred.mapper.new-api</name>
                <value>true</value>
            </property>

            <property>
                <name>mapred.reducer.new-api</name>
                <value>true</value>
                 </property>

            <property>
                <name>mapreduce.map.class</name>
                <value>WordCount.Map</value>
            </property>

            <property>
                <name>mapreduce.reduce.class</name>
                <value>WordCount.Reduce</value>
            </property>

            <property>
              <name>mapred.output.key.class</name>
              <value>org.apache.hadoop.io.Text</value>
            </property>

            <property>
               <name>mapred.output.value.class</name>
               <value>org.apache.hadoop.io.IntWritable</value>
            </property>

            <property>
                <name>mapred.map.tasks</name>
                <value>1</value>
            </property>

            <property>
                <name>mapred.input.dir</name>
                <value>/user/${wf:user()}/${wordcountRoot}/input-data</value>
            </property>
            <property>
                <name>mapred.output.dir</name>
                <value>/user/${wf:user()}/${wordcountRoot}/output-data/${outputDir}</value>
            </property>

            <property>
                <name>mapred.job.queue.name</name>
                <value>${queueName}</value>
            </property>

            <property>
             <name>mapreduce.job.acl-view-job</name>
             <value>*</value>
            </property>

            <property>
               <name>oozie.launcher.mapreduce.job.acl-view-job</name>
               <value>*</value>
            </property>

         </configuration>

    </map-reduce>

    <ok to="end" />
    <error to="fail" />
</action>

<kill name="fail">
    <message>Map/Reduce failed</message>
</kill>
<end name="end" />

我提到了这个链接https://cwiki.apache.org/OOZIE/map-reduce-cookbook.html，但仍然没有运气。如果any1遇到过这个问题，请指导我哪里出错了。

提前致谢。

Answer 1

问题解决了...... 在使用新的mapreduce API时，我们需要在＆＃34; $＆＃34;前加上前缀。 mapper和reducer类名称的符号：

<property>
 <name>mapreduce.map.class</name>
 <value>oozie.WordCount$Map</value>
</property> 
<property> 
 <name>mapreduce.reduce.class</name>
 <value>oozie.WordCount$Reduce</value>
</property>

将Map-Reduce与Oozie集成时出错

1 个答案: