如何在hadoop mapreduce中实现自适应合并排序

时间:2017-03-31 20:32:15

标签: java hadoop

我在java中编写了自适应合并排序。但作为mapreduce程序,有三个类。一个是地图,两个是减速器,第三个是驱动器。我不明白如何将此代码转换为mapreduce,以便我可以在hadoop多节点集群上运行它?

这是代码:

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Scanner;

public class MainClass {

    public static void main(String args[]) throws Exception {

        MainClass mainClass = new MainClass();
        MainClass.SortClass sortClass = mainClass.new SortClass();
        sortClass.sortMainFunction();
    }

    public class Constant {
        public static final int MAX = 10000;
        public static final int max = 10000;
    }

    public class Values {
        int st_index;
        int ed_index;
        boolean as_ds;

        public Values(int st_index, int ed_index, boolean as_ds) {
            this.st_index = st_index;
            this.ed_index = ed_index;
            this.as_ds = as_ds;
        }
    }

    public class SortClass {
        int[] a = new int[Constant.MAX];
        int[] b = new int[Constant.MAX];
        int index1 = 0;
        int i = 0;
        int numberOfItem = 0, first_ind, end_ind, tem, k, g, h;
        boolean flag;
        Values node[] = new Values[Constant.max];
        int p, q, r, low, high, j;

        void insert(int st_ind, int end_ind, boolean fl) {


            node[index1] = new Values(st_ind, end_ind, fl);
            index1++;
        }

        public void sortMainFunction() throws Exception {
            addDataIntoArray();

            first_ind = 0;
            end_ind = 0;
            flag = false;
            tem = a[0];
            k = 1;

            while (true) {

                while (true) {
                    if (k < i && tem >= a[k]) {
                        end_ind = k;
                        tem = a[k];
                        k++;
                        flag = true;
                        continue;
                    }
                    break;
                }
                if (flag) {



                    node[index1] = new Values(first_ind, end_ind, flag);
                    index1++;


                    first_ind = k;
                    end_ind = k;
                    if (k >= i) {

                    } else {
                        tem = a[k++];
                    }

                }

                while (true) {
                    if (k < i && tem <= a[k]) {
                        tem = a[k];
                        end_ind = k;
                        flag = false;
                        k++;
                        continue;
                    }
                    break;
                }
                if (!flag) {
                    insert(first_ind, end_ind, flag);
                    first_ind = end_ind = k;
                    if (k >= i) {

                    } else {
                        tem = a[k++];
                    }

                }

                if (end_ind == i)
                    break;
            }

            g = index1 - 1;
            h = 0;
            while (true) {
                if (g / 2 <= 0) {
                    break;
                }
                g /= 2;
                h++;
            }

            System.out.println(h);
            h = (int) ((double) Math.log((double) index1 - 1) / (double) Math.log(2.0));
            System.out.println(h);

            p = 1;
            for (q = 0; q <= h; q++, p *= 2) {
                for (r = 0; r < index1; r++) {
                    if (2 * r * p + p >= index1)
                        break;
                    else {
                        low = 2 * p * r;
                        high = 2 * p * r + p;
                        k = node[low].st_index;
                        if (node[low].as_ds == false && node[high].as_ds == false) {
                            i = node[low].st_index;
                            j = node[high].st_index;

                            while (i <= node[low].ed_index && j <= node[high].ed_index) {
                                if (a[i] <= a[j])
                                    b[k++] = a[i++];
                                else
                                    b[k++] = a[j++];
                            }
                            if (i > node[low].ed_index)
                                while (j <= node[high].ed_index)
                                    b[k++] = a[j++];
                            else
                                while (i <= node[low].ed_index)
                                    b[k++] = a[i++];
                        } else if (node[low].as_ds == false && node[high].as_ds == true) {
                            i = node[low].st_index;
                            j = node[high].ed_index;

                            while (i <= node[low].ed_index && j >= node[high].st_index) {
                                if (a[i] <= a[j])
                                    b[k++] = a[i++];
                                else
                                    b[k++] = a[j--];
                            }
                            if (i > node[low].ed_index)
                                while (j >= node[high].st_index)
                                    b[k++] = a[j--];
                            else
                                while (i <= node[low].ed_index)
                                    b[k++] = a[i++];
                        } else if (node[low].as_ds == true && node[high].as_ds == false) {
                            i = node[low].ed_index;
                            j = node[high].st_index;

                            while (i >= node[low].st_index && j <= node[high].ed_index) {
                                if (a[i] <= a[j])
                                    b[k++] = a[i--];
                                else
                                    b[k++] = a[j++];
                            }
                            if (j > node[high].ed_index)
                                while (i >= node[low].st_index)
                                    b[k++] = a[i--];
                            else
                                while (j <= node[high].ed_index)
                                    b[k++] = a[j++];
                        } else if (node[low].as_ds == true && node[high].as_ds == true) {
                            i = node[low].ed_index;
                            j = node[high].ed_index;

                            while (i >= node[low].st_index && j >= node[high].st_index) {
                                if (a[i] <= a[j])
                                    b[k++] = a[i--];
                                else
                                    b[k++] = a[j--];
                            }
                            if (i < node[low].st_index)
                                while (j >= node[high].st_index)
                                    b[k++] = a[j--];
                            else
                                while (i >= node[low].st_index)
                                    b[k++] = a[i--];
                        }
                        for (k = node[low].st_index; k <= node[high].ed_index; k++)
                            a[k] = b[k];

                        node[low].ed_index = node[high].ed_index;
                        node[high].st_index = node[low].st_index;
                        node[low].as_ds = false;
                        node[high].as_ds = false;
                    }
                }
            }



            BufferedWriter output = null;

            File file = new File("output.txt");
            output = new BufferedWriter(new FileWriter(file));
            for (k = 0; k < numberOfItem; k++) {
                String s = String.valueOf(a[k]);
                output.write(s);
                output.newLine();
            }

            if (output != null) {
                output.close();
            }



        }

        public void addDataInt

    oArray() throws Exception {
                Scanner scanner;
                scanner = new Scanner(new File("input.txt"));
                while (scanner.hasNextInt()) {
                    a[i] = scanner.nextInt();
                    i++;
                    numberOfItem++;
                }

            }
        }

    }

0 个答案:

没有答案