MapReduce客户端在作业完成后重试连接

时间:2016-07-31 05:53:58

标签: hadoop apache-pig

在Hadoop 2.6.0-cdh5.7.0上运行并发出一个简单的Pig脚本 成功完成作业后,我收到以下消息:

似乎工作者正在尝试彼此通信(最多重试3次),但我不确定原因,以及配置此行为的位置。

有谁知道如何解决这个问题?

       class ConditionalMap extends HashMap {

        /** Default condition can be a closure */
        def defaultCondition = true

        /** Put an elemtn with default condition */
        def put(key, value) {
            super.put(key, new Tuple(defaultCondition, value))
        }

        /** Put an elemetn with specific condition */
        def put(key, value, condition) {
            super.put(key, new Tuple(condition, value))
        }

        /** Get visible element only */
        def get(key) {
            def tuple = super.get(key)
            tuple[0] == true ? tuple[1] : null
        }

        /** Not part of Map , just to know the real size*/
        def int realSize() {
            super.keySet().size()
        }

        /** Includes only the "visible" elements keys */
        def Set keySet() {
            super.keySet().inject(new HashSet(),
                    { result, key
                        ->
                        def tuple = super.get(key)
                        if (tuple[0])
                            result.add(key)
                        result
                    })
        }

        /** Includes only the "visible" elements keys */
        def Collection values() {
            this.keySet().asCollection().collect({ k -> this[k] })
        }

        /** Includes only the "visible" elements keys */
        def int size() {
            this.keySet().size()
        }
    }

    /** default condition that do not accept elements */
    def map = new ConditionalMap(defaultCondition: false)

    /** condition can be a closure too */
    // def map = new ConditionalMap(defaultCondition : {-> true == false })


    map["a"] = "A"
    map["b"] = "B"
    map.put("c","C", true)
    map.put("d","D", false)

    assert map.size() == 1
    assert map.realSize() == 4

    println map["a"]
    println map["b"]
    println map["c"]
    println map["d"]

    println "size: ${map.size()}"
    println "realSize: ${map.realSize()}"
    println "keySet: ${map.keySet()}"
    println "values: ${map.values()}"

    /** end of script */

0 个答案:

没有答案