Mongo似乎已经用光了！？我该怎么办？
我甚至不确定它实际上是一个记忆问题。
2016-08-11T14:23:26.875+0200 E CONTROL [conn2575] mmap private failed with out of memory. (64 bit build)
2016-08-11T14:23:26.875+0200 I - [conn2575] Assertion: 13636:file /data/db/project.6 open/create failed in createPrivateMap (look in log for more information)
2016-08-11T14:23:26.892+0200 I CONTROL [conn2575]
0xf9b632 0xf36699 0xf1a22e 0xf1a2dc 0xd56925 0xd56b5f 0xd3cf06 0xd6ff6d 0xd70380 0xd7061d 0xd74ec2 0xd80b34 0xd756ab 0xd75859 0x90a811 0x90b3ec 0xa3e5fc 0xa3f334 0xbe8012 0xbe832b 0xbe8a0d 0x9b1769 0x9b2fc5 0x9b35f5 0x9b62dc 0x9d9071 0x9da07c 0x9dad4b 0xbb544b 0xac3e4a 0x7fd9c5 0xf4af79 0x7f9122c60aa1 0x7f91217b293d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B9B632","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B36699","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B1A22E","s":"_ZN5mongo11msgassertedEiPKc"},{"b":"400000","o":"B1A2DC"},{"b":"400000","o":"956925","s":"_ZN5mongo17DurableMappedFile13finishOpeningEv"},{"b":"400000","o":"956B5F","s":"_ZN5mongo17DurableMappedFile6createERKSsRyb"},{"b":"400000","o":"93CF06","s":"_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib"},{"b":"400000","o":"96FF6D","s":"_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib"},{"b":"400000","o":"970380","s":"_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib"},{"b":"400000","o":"97061D","s":"_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib"},{"b":"400000","o":"974EC2","s":"_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib"},{"b":"400000","o":"980B34","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"9756AB","s":"_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"975859","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"50A811","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"50B3EC","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"63E5FC","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"63F334","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"7E8012","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"7E832B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"7E8A0D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5B1769","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5B2FC5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5B35F5","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE"},{"b":"400000","o":"5B62DC","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5D9071","s":"_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DA07C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DAD4B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7B544B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"400000","o":"6C3E4A","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"3FD9C5","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B4AF79","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"7F9122C59000","o":"7AA1"},{"b":"7F91216CA000","o":"E893D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.9", "gitVersion" : "20d60d3491908f1ae252fe452300de3978a040c7", "uname" : { "sysname" : "Linux", "release" : "2.6.32-042stab113.11", "version" : "#1 SMP Fri Dec 18 17:32:04 MSK 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "47DFD0896283C6BF009395E08B673D3246CC7AE5" }, { "b" : "7FFD9BE1E000", "elfType" : 3, "buildId" : "F466F3FAB4F73E9911206E2486A3C5F8FCCA8EA5" }, { "b" : "7F9122C59000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "C56DD1B811FC0D9263248EBB308C73FCBCD80FC1" }, { "b" : "7F91229ED000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "24A4E7055CABC29C2DDAC730D635A20C08687AC6" }, { "b" : "7F912260A000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F6F5BB27A852F29A134680794DD2EB5C3CC5D88B" }, { "b" : "7F9122402000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "95159178F1A4A3DBDC7819FBEA2C80E5FCDD6BAC" }, { "b" : "7F91221FE000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "29B61382141595ECBA6576232E44F2310C3AAB72" }, { "b" : "7F9121EF8000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "b" : "7F9121C74000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "b" : "7F9121A5E000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "b" : "7F91216CA000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "8E6FA4C4B0594C355C1B90C1D49990368C81A040" }, { "b" : "7F9122E76000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "959C5E10A47EE8A633E7681B64B4B9F74E242ED5" }, { "b" : "7F9121486000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "b" : "7F912119F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "b" : "7F9120F9B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "b" : "7F9120D6F000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "b" : "7F9120B59000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F912094E000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "b" : "7F912074B000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F9120531000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C39D7FFB49DFB1B55AD09D1D711AD802123F6623" }, { "b" : "7F9120312000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xf9b632]
mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf36699]
mongod(_ZN5mongo11msgassertedEiPKc+0xAE) [0xf1a22e]
mongod(+0xB1A2DC) [0xf1a2dc]
mongod(_ZN5mongo17DurableMappedFile13finishOpeningEv+0x2F5) [0xd56925]
mongod(_ZN5mongo17DurableMappedFile6createERKSsRyb+0x6F) [0xd56b5f]
mongod(_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib+0x106) [0xd3cf06]
mongod(_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib+0x15D) [0xd6ff6d]
mongod(_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib+0xE0) [0xd70380]
mongod(_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib+0x1FD) [0xd7061d]
mongod(_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib+0x42) [0xd74ec2]
mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x1F4) [0xd80b34]
mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd756ab]
mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd75859]
mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x90a811]
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x90b3ec]
mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa3e5fc]
mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa3f334]
mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xbe8012]
mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xbe832b]
mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbe8a0d]
mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9b1769]
mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x245) [0x9b2fc5]
mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9b35f5]
mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9b62dc]
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9d9071]
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9da07c]
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9dad4b]
mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbb544b]
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xac3e4a]
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x7fd9c5]
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf4af79]
libpthread.so.0(+0x7AA1) [0x7f9122c60aa1]
libc.so.6(clone+0x6D) [0x7f91217b293d]
----- END BACKTRACE -----
ulimit -a
core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimited
scheduling priority (-e) 0
file size (blocks, -f) unlimited
pending signals (-i) 127948
max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0
stack size (kbytes, -s) 10240
cpu time (seconds, -t) unlimited
max user processes (-u) 127948
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited
ulimit -v
unlimited
swapon -s
Filename Type Size Used Priority
/dev/null partition 524288 175344 -1
free -m
total used free shared buffers cached
Mem: 4096 3331 764 15 0 2985
-/+ buffers/cache: 345 3750
Swap: 512 171 340
df -h
Filesystem Size Used Avail Use% Mounted on
/dev/ploop10759p1 40G 26G 12G 69% /
none 16G 4.0K 16G 1% /dev
none 16G 0 16G 0% /dev/shm
# cat /proc/sys/vm/overcommit_memory
0
ps aux | grep mongo
webdev 4006 0.2 26.0 9123780 1091292 ? Sl 14:54 0:06 mongod --logpath /var/log/mongodb/mongod.log
webdev 6031 0.0 0.0 103312 872 pts/1 S+ 15:35 0:00 grep mongo
mongod --version
db version v3.0.12
git version: 33934938e0e95d534cebbaff656cde916b9c3573
cat / etc / centos-release
CentOS release 6.8 (Final)
db.serverStatus（）
{
"host" : "host",
"version" : "3.0.12",
"process" : "mongod",
"pid" : NumberLong(4006),
"uptime" : 3590,
"uptimeMillis" : NumberLong(3590668),
"uptimeEstimate" : 3555,
"localTime" : ISODate("2016-08-11T13:54:51.869Z"),
"asserts" : {
"regular" : 0,
"warning" : 1,
"msg" : 0,
"user" : 0,
"rollovers" : 0
},
"backgroundFlushing" : {
"flushes" : 59,
"total_ms" : 80,
"average_ms" : 1.3559322033898304,
"last_ms" : 1,
"last_finished" : ISODate("2016-08-11T13:54:01.250Z")
},
"connections" : {
"current" : 4,
"available" : 815,
"totalCreated" : NumberLong(6)
},
"cursors" : {
"note" : "deprecated, use server status metrics",
"clientCursors_size" : 0,
"totalOpen" : 0,
"pinned" : 0,
"totalNoTimeout" : 0,
"timedOut" : 0
},
"dur" : {
"commits" : 24,
"journaledMB" : 0,
"writeToDataFilesMB" : 0,
"compression" : 0,
"commitsInWriteLock" : 0,
"earlyCommits" : 0,
"timeMs" : {
"dt" : 0,
"prepLogBuffer" : 0,
"writeToJournal" : 0,
"writeToDataFiles" : 0,
"remapPrivateView" : 0,
"commits" : 0,
"commitsInWriteLock" : 0
}
},
"extra_info" : {
"note" : "fields vary by platform",
"heap_usage_bytes" : 63466248,
"page_faults" : 28
},
"globalLock" : {
"totalTime" : NumberLong("3590669000"),
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 12,
"readers" : 0,
"writers" : 0
}
},
"locks" : {
"Global" : {
"acquireCount" : {
"r" : NumberLong(47395),
"w" : NumberLong(1270),
"W" : NumberLong(5)
},
"acquireWaitCount" : {
"r" : NumberLong(1)
},
"timeAcquiringMicros" : {
"r" : NumberLong(641)
}
},
"MMAPV1Journal" : {
"acquireCount" : {
"r" : NumberLong(23058),
"w" : NumberLong(1300),
"R" : NumberLong(35093)
},
"acquireWaitCount" : {
"w" : NumberLong(3),
"R" : NumberLong(3)
},
"timeAcquiringMicros" : {
"w" : NumberLong(24),
"R" : NumberLong(63215)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(23056),
"w" : NumberLong(1247),
"R" : NumberLong(4),
"W" : NumberLong(23)
}
},
"Collection" : {
"acquireCount" : {
"R" : NumberLong(24059),
"W" : NumberLong(1247)
}
},
"Metadata" : {
"acquireCount" : {
"W" : NumberLong(1)
}
}
},
"network" : {
"bytesIn" : NumberLong(28574),
"bytesOut" : NumberLong(14793714),
"numRequests" : NumberLong(229)
},
"opcounters" : {
"insert" : 0,
"query" : 216,
"update" : 1,
"delete" : 0,
"getmore" : 3,
"command" : 11
},
"opcountersRepl" : {
"insert" : 0,
"query" : 0,
"update" : 0,
"delete" : 0,
"getmore" : 0,
"command" : 0
},
"storageEngine" : {
"name" : "mmapv1"
},
"writeBacksQueued" : false,
"mem" : {
"bits" : 64,
"resident" : 1068,
"virtual" : 8909,
"supported" : true,
"mapped" : 4287,
"mappedWithJournal" : 8574
},
"metrics" : {
"commands" : {
"getLog" : {
"failed" : NumberLong(0),
"total" : NumberLong(1)
},
"isMaster" : {
"failed" : NumberLong(0),
"total" : NumberLong(5)
},
"replSetGetStatus" : {
"failed" : NumberLong(1),
"total" : NumberLong(1)
},
"serverStatus" : {
"failed" : NumberLong(0),
"total" : NumberLong(2)
},
"update" : {
"failed" : NumberLong(0),
"total" : NumberLong(1)
},
"whatsmyuri" : {
"failed" : NumberLong(0),
"total" : NumberLong(2)
}
},
"cursor" : {
"timedOut" : NumberLong(0),
"open" : {
"noTimeout" : NumberLong(0),
"pinned" : NumberLong(0),
"total" : NumberLong(0)
}
},
"document" : {
"deleted" : NumberLong(0),
"inserted" : NumberLong(0),
"returned" : NumberLong(26915),
"updated" : NumberLong(0)
},
"getLastError" : {
"wtime" : {
"num" : 0,
"totalMillis" : 0
},
"wtimeouts" : NumberLong(0)
},
"operation" : {
"fastmod" : NumberLong(0),
"idhack" : NumberLong(0),
"scanAndOrder" : NumberLong(140),
"writeConflicts" : NumberLong(0)
},
"queryExecutor" : {
"scanned" : NumberLong(599),
"scannedObjects" : NumberLong(439045)
},
"record" : {
"moves" : NumberLong(0)
},
"repl" : {
"apply" : {
"batches" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0)
},
"buffer" : {
"count" : NumberLong(0),
"maxSizeBytes" : 268435456,
"sizeBytes" : NumberLong(0)
},
"network" : {
"bytes" : NumberLong(0),
"getmores" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0),
"readersCreated" : NumberLong(0)
},
"preload" : {
"docs" : {
"num" : 0,
"totalMillis" : 0
},
"indexes" : {
"num" : 0,
"totalMillis" : 0
}
}
},
"storage" : {
"freelist" : {
"search" : {
"bucketExhausted" : NumberLong(0),
"requests" : NumberLong(1),
"scanned" : NumberLong(0)
}
}
},
"ttl" : {
"deletedDocuments" : NumberLong(0),
"passes" : NumberLong(59)
}
},
"ok" : 1
}
答案 0 :(得分：1)
看起来您目前在1024打开文件时受到限制。虽然伴随这种失败通常会出现助推错误，但值得尝试像ulimit -n 4096
答案 1 :(得分：0)
增加ram后问题消失了。 它可能会在某个时候再次出现。 至少现在我知道，如果MongoDB出现问题，即使在stackoverflow上也很难找到答案。