如何提高mongodb map / reduce的性能

时间:2016-03-13 07:59:35

标签: performance mongodb mapreduce

我的mongodb群集使用3个分片,其中包含7000万条记录。

硬件:16g内存。

我想使用下面的地图/减少进度来做一些计算。

db.runCommand({
    mapReduce: 'orders',
    map: function(){
        var key = { 'name': this.Receiver_name, 'mobile': this.Receiver_mobile };
        var values = { 'count': 1, 'dates': [ this.Receiver_date ] };
        emit(key, values);
    },
    reduce: function(key, values){
        var result = { count: 0, dates: 0 };
        var dates = [];      
        values.forEach(function(value){
            result.count += value.count;
            dates = dates.concat(value.dates);
        });
        result.dates = new Date(Math.min.apply(Math, dates));
        return result;
    },
    sort: { Receiver_name: 1, Receiver_mobile: 1 }, //do index
    out:  { replace: 'localtest' }
})

我使用How to speed up MongoDB Map Reduce by 20x引用了一些建议,而没有使用多线程和多数据库。由于mongodb测试2.6.11上的多线程mising,我不能使用ScopedThread()函数。

所以结果仍然是

db.currentOp()

下面是当前操作日志:S1完成,在地图阶段通知S3仅2%。

{
    "inprog" : [ 
        {
            "opid" : "s3:99457482",
            "active" : true,
            "secs_running" : 1644,
            "microsecs_running" : NumberLong(1644096961),
            "op" : "query",
            "ns" : "express.orders",
            "query" : {
                "$msg" : "query not recording (too large)"
            },
            "client_s" : "222.31.79.193:36487",
            "desc" : "conn41",
            "threadId" : "0x7fc4ff361700",
            "connectionId" : 41,
            "waitingForLock" : false,
            "msg" : "m/r: (1/3) emit phase M/R: (1/3) Emit Progress: 740369/29745378 2%",
            "progress" : {
                "done" : 740369,
                "total" : 29745378
            },
            "numYields" : 213942,
            "lockStats" : {
                "timeLockedMicros" : {
                    "r" : NumberLong(299591221),
                    "w" : NumberLong(1368691)
                },
                "timeAcquiringMicros" : {
                    "r" : NumberLong(708109),
                    "w" : NumberLong(91251)
                }
            }
        }, 
        {
            "opid" : "s2:158299848",
            "active" : true,
            "secs_running" : 1644,
            "microsecs_running" : NumberLong(1644123918),
            "op" : "query",
            "ns" : "express.orders",
            "query" : {
                "$msg" : "query not recording (too large)"
            },
            "client_s" : "222.31.79.193:48366",
            "desc" : "conn2332",
            "threadId" : "0x7f850d9a6700",
            "connectionId" : 2332,
            "locks" : {
                "^" : "r",
                "^express" : "R"
            },
            "waitingForLock" : false,
            "msg" : "m/r: (1/3) emit phase M/R: (1/3) Emit Progress: 28830696/30690385 93%",
            "progress" : {
                "done" : 28830696,
                "total" : 30690385
            },
            "numYields" : 288816,
            "lockStats" : {
                "timeLockedMicros" : {
                    "r" : NumberLong(2522717442),
                    "w" : NumberLong(56346161)
                },
                "timeAcquiringMicros" : {
                    "r" : NumberLong(750081),
                    "w" : NumberLong(8547462)
                }
            }
        }
    ]
}

0 个答案:

没有答案