Question

我正在调用cuda代码来获取每个键的所有值的总和。目的是通过并联操作来减少减速器所需的时间。但是，reducer中的值是IntWritable形式。所以，我必须将它们转换为一个整数数组，以传递给cuda代码。这是我的reducer代码：

public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
   private IntWritable result = new IntWritable();

   public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
       List<Integer> numbers = new ArrayList<Integer>();
       for(IntWritable val : values)
            numbers.add(val.get());
       }
       int[] ret = ArrayUtils.toPrimitive(numbers.toArray(new Integer[numbers.size()]));
       result.set(Main.sumNumbers(ret));
       context.write(key,result);
   }
}

问题是，为了将IntWritable转换为Integer数组，我必须遍历每个值，这是一个串行操作。所以，它正在增加时间。那么，有什么办法可以让我不必遍历每个值并直接转换为int数组？

以下是映射器代码：

public static class TokenizerMapper extends
            Mapper<Object, Text, Text, IntWritable> {

        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        public void map(Object key, Text value, Context context)
                throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while (itr.hasMoreTokens()) {
                word.set(itr.nextToken());
                context.write(word, one);
            }
        }
    }

这是我的cuda代码：

#include <stdio.h>
#ifndef _ADDARRAY_KERNEL_H_
#define _ADDARRAY_KERNEL_H_

#ifdef __cplusplus
extern "C"
{
#endif
__global__ void add_array(int *a, int *c, int N)
{
  *c = 0;
  int i;
   for(i = 0; i<N;i++)
   {
    *c = *c + a[i];
   }
}
#ifdef __cplusplus
}
#endif 
#endif // #ifndef _ADDARRAY_KERNEL_H_

#ifdef __cplusplus
extern "C"
{
#endif

int cuda_sum(int *a_h, int N)
{   
    int *a_d, c=0;
    int *dev_c;
    cudaMalloc((void**)&dev_c, sizeof(int));
    size_t size = N * sizeof (int);

//      a_h = (int *) malloc(size);
    cudaMalloc((void **) & a_d, size);
    cudaMemcpy(a_d, a_h, size, cudaMemcpyHostToDevice);
        add_array <<<1, 1 >>>(a_d, dev_c, N);
        cudaMemcpy(&c, dev_c, sizeof(int), cudaMemcpyDeviceToHost);
    cudaFree(dev_c);
    return c;
}
#ifdef __cplusplus
}
#endif

由于

Answer 1

我建议您应该采取以下措施

public static class IntSumReducer extends Reducer<Text, IntWritable, Text, ArrayPrimitiveWritable>{ private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable<ArrayPrimitiveWritable> values, Context context) throws IOException,InterruptedException { int[] ret = values.next(); result.set(Main.sumNumbers(ret)); context.write(key,result); } }

ArrayPrimitiveWritable 将为您完成这项工作。

Answer 2

public static class IntSumReducer extends Reducer<Text, ArrayPrimitiveWritable, Text, IntWritable>{
        private IntWritable result = new IntWritable();
        public void reduce(Text key, Iterable<ArrayPrimitiveWritable> values, Context context) throws IOException,InterruptedException {
            ArrayPrimitiveWritable arrayOfInts = values.iterator().next();
            final int[] ret = (int [])arrayOfInts.get();
            result.set(Main.sumNumbers(ret));
            context.write(key,result);
        }
    }

我稍微修改了代码以便它可以正常工作，但是我还没有运行它，你能不能试一试。我坚持使用刚刚进行更新的ArrayPrimitiveWritable，以免编译错误消失。

在Hadoop＆＃39的Reducer中获取整数数组

2 个答案: