我的mongodb群集使用3个分片,其中包含7000万条记录。
硬件:16g内存。
我想使用下面的地图/减少进度来做一些计算。
db.runCommand({
mapReduce: 'orders',
map: function(){
var key = { 'name': this.Receiver_name, 'mobile': this.Receiver_mobile };
var values = { 'count': 1, 'dates': [ this.Receiver_date ] };
emit(key, values);
},
reduce: function(key, values){
var result = { count: 0, dates: 0 };
var dates = [];
values.forEach(function(value){
result.count += value.count;
dates = dates.concat(value.dates);
});
result.dates = new Date(Math.min.apply(Math, dates));
return result;
},
sort: { Receiver_name: 1, Receiver_mobile: 1 }, //do index
out: { replace: 'localtest' }
})
我使用How to speed up MongoDB Map Reduce by 20x引用了一些建议,而没有使用多线程和多数据库。由于mongodb测试2.6.11上的多线程mising,我不能使用ScopedThread()函数。
所以结果仍然是
下面是当前操作日志:S1完成,在地图阶段通知S3仅2%。
{
"inprog" : [
{
"opid" : "s3:99457482",
"active" : true,
"secs_running" : 1644,
"microsecs_running" : NumberLong(1644096961),
"op" : "query",
"ns" : "express.orders",
"query" : {
"$msg" : "query not recording (too large)"
},
"client_s" : "222.31.79.193:36487",
"desc" : "conn41",
"threadId" : "0x7fc4ff361700",
"connectionId" : 41,
"waitingForLock" : false,
"msg" : "m/r: (1/3) emit phase M/R: (1/3) Emit Progress: 740369/29745378 2%",
"progress" : {
"done" : 740369,
"total" : 29745378
},
"numYields" : 213942,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(299591221),
"w" : NumberLong(1368691)
},
"timeAcquiringMicros" : {
"r" : NumberLong(708109),
"w" : NumberLong(91251)
}
}
},
{
"opid" : "s2:158299848",
"active" : true,
"secs_running" : 1644,
"microsecs_running" : NumberLong(1644123918),
"op" : "query",
"ns" : "express.orders",
"query" : {
"$msg" : "query not recording (too large)"
},
"client_s" : "222.31.79.193:48366",
"desc" : "conn2332",
"threadId" : "0x7f850d9a6700",
"connectionId" : 2332,
"locks" : {
"^" : "r",
"^express" : "R"
},
"waitingForLock" : false,
"msg" : "m/r: (1/3) emit phase M/R: (1/3) Emit Progress: 28830696/30690385 93%",
"progress" : {
"done" : 28830696,
"total" : 30690385
},
"numYields" : 288816,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(2522717442),
"w" : NumberLong(56346161)
},
"timeAcquiringMicros" : {
"r" : NumberLong(750081),
"w" : NumberLong(8547462)
}
}
}
]
}