hbase协处理器无法从shell加载

时间:2016-03-15 09:44:56

标签: hadoop hbase

我正在尝试添加一个带有一个hbase表的协处理器,但它失败并出现错误 -

2016-03-15 14:40:14,130 INFO org.apache.hadoop.hbase.regionserver.RSRpcServices: Open PRODUCT_DETAILS,,1457953190424.f687dd250bfd1f18ffbb8075fd625145.
2016-03-15 14:40:14,173 ERROR org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost: Failed to load coprocessor com.optymyze.coprocessors.ProductObserver
java.io.IOException: Failed on local exception: com.google.protobuf.InvalidProtocolBufferException: Protocol message end-group tag did not match expected tag.; Host Details : local host is: "mylocalhost/mylocalhostip"; destination host is: "mydestinationhost":9000; 
    at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:772)

添加我执行的co处理器 -

  • HBase的>禁用“PRODUCT_DETAILS”
  • HBase的&GT;改变'PRODUCT_DETAILS',METHOD =&gt; 'table_att','coprocessor'=&gt;'hdfs:// mydestinationhost:9000 / hbase-coprocessors-0.0.3-SNAPSHOT.jar | com.optymyze.coprocessors.ProductObserver | 1001 | arg1 = 1,arg2 = 2'< / LI>
  • 现在启用'PRODUCT_DETAILS'将无效。

协处理器代码如下 -

 package com.optymyze.coprocessors;

import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.slf4j.LoggerFactory.*;

/**
 *
 * Created by adnan on 14-03-2016.
 */
public class ProductObserver extends BaseRegionObserver {

    private static final Logger LOGGER = getLogger(ProductObserver.class);

    private static final String PRODUCT_DETAILS_TABLE = "PRODUCT_DETAILS";
    public static final String COLUMN_FAMILY = "CF";

    @Override
    public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, boolean writeToWAL) throws IOException {
        List<KeyValue> kvs = put.getFamilyMap().get(Bytes.toBytes(COLUMN_FAMILY));
        LOGGER.info("key values {}", kvs);
        Map<String, Integer> qualifierVsValue = getMapForQualifierVsValuesForRequiredOnes(kvs);
        LOGGER.info("qualifier values {}", qualifierVsValue);
        List<Put> puts = createPuts(kvs, qualifierVsValue);
        LOGGER.info("puts values {}", puts);
        updateProductTable(e, puts);
        LOGGER.info("puts done");
    }

    private void updateProductTable(ObserverContext<RegionCoprocessorEnvironment> e, List<Put> puts) throws IOException {
        HTableInterface productTable = e.getEnvironment().getTable(Bytes.toBytes(PRODUCT_DETAILS_TABLE));
        try {
            productTable.put(puts);
        }finally {
            productTable.close();
        }
    }

    private List<Put> createPuts(List<KeyValue> kvs, Map<String, Integer> qualifierVsValue) {
        int salePrice, baseline = 0, finalPrice = 0;
        List<Put> puts = new ArrayList<Put>(kvs.size());
        for (KeyValue kv : kvs) {
            if (kv.matchingQualifier(Bytes.toBytes("BASELINE"))) {
                baseline = convertToZeroIfNull(qualifierVsValue, "PRICE")
                        - convertToZeroIfNull(qualifierVsValue, "PRICE")
                        * convertToZeroIfNull(qualifierVsValue, "DISCOUNT") / 100;
                puts.add(newPut(kv, baseline));
            }
            if (kv.matchingQualifier(Bytes.toBytes("FINALPRICE"))) {
                finalPrice = baseline + baseline * convertToZeroIfNull(qualifierVsValue, "UPLIFT") / 100;
                puts.add(newPut(kv, finalPrice));
            }

            if (kv.matchingQualifier(Bytes.toBytes("SALEPRICE"))) {
                salePrice = finalPrice * convertToZeroIfNull(qualifierVsValue, "VOLUME");
                puts.add(newPut(kv, salePrice));
            }
        }
        return puts;
    }

    private Map<String, Integer> getMapForQualifierVsValuesForRequiredOnes(List<KeyValue> kvs) {
        Map<String, Integer> qualifierVsValue = new HashMap<String, Integer>();
        for (KeyValue kv : kvs) {
            getValueFromQualifier(kv, "PRICE", qualifierVsValue);
            getValueFromQualifier(kv, "DISCOUNT", qualifierVsValue);
            getValueFromQualifier(kv, "UPLIFT", qualifierVsValue);
            getValueFromQualifier(kv, "VOLUME", qualifierVsValue);
        }
        return qualifierVsValue;
    }

    private Integer convertToZeroIfNull(Map<String, Integer> qualifierVsValue, String qualifier) {
        Integer v = qualifierVsValue.get(qualifier);
        return v == null ? 0 : v;
    }

    private void getValueFromQualifier(KeyValue kv, String qualifier, Map<String, Integer> qualifierVsValue) {
        if (kv.matchingQualifier(Bytes.toBytes(qualifier))) {
            qualifierVsValue.put(qualifier, Bytes.toInt(convertToByteZeroIfNull(kv)));
        }
    }

    private Put newPut(KeyValue kv, int newVal) {
        Put put = new Put(kv.getValue(), kv.getTimestamp());
        put.add(kv.getFamily(), kv.getQualifier(), Bytes.toBytes(newVal));
        return put;
    }

    private byte[] convertToByteZeroIfNull(KeyValue kv) {
        return kv.getValue() == null ? Bytes.toBytes(0) : kv.getValue();
    }
}

0 个答案:

没有答案