I have been facing continuous RAM memory depletion on mongo hosts.
After exploring it was found that mongodb tcmalloc natural behaviour is memory hungry however did not find specifics related to the threshold at which mongod will start releasing RAM.
mongo_java_driver version we are using is 3.7.1
We conducted a test to run a python script consuming 1gb of ram memory (with lesser nice value e.g. -16 than mongod which is running at -15) and found that the python process got killed by OOM.
I have below 2 queries:
Will mongodb wait for a threshold to release the memory, if yes then what is that threshold? OR there is no such threshold and it keeps hogging RAM and eventually ends up getting killed by OOM?
I also observed that tcmallocReleaseRate was introduced however https://jira.mongodb.org/browse/SERVER-42697 mentions that even that is also not turning out to be beneficial so is there safer way apart from dropCache which ensures ram release with minimum side effect?
rs-shard-34:PRIMARY> db.serverStatus()
{
"host" : "mongo-s105:27041",
"version" : "3.6.9",
"process" : "mongod",
"pid" : NumberLong(3035),
"uptime" : 351477,
"uptimeMillis" : NumberLong(351477511),
"uptimeEstimate" : NumberLong(351477),
"localTime" : ISODate("2020-04-11T17:20:51.856Z"),
"asserts" : {
"regular" : 0,
"warning" : 0,
"msg" : 0,
"user" : 26,
"rollovers" : 0
},
"backgroundFlushing" : {
"flushes" : 5857,
"total_ms" : 498,
"average_ms" : 0.08502646406009903,
"last_ms" : 0,
"last_finished" : ISODate("2020-04-11T17:19:55.014Z")
},
"connections" : {
"current" : 360,
"available" : 838500,
"totalCreated" : 30878
},
"extra_info" : {
"note" : "fields vary by platform",
"page_faults" : 636
},
"globalLock" : {
"totalTime" : NumberLong("351477518000"),
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 415,
"readers" : 0,
"writers" : 0
}
},
"locks" : {
"Global" : {
"acquireCount" : {
"r" : NumberLong(1253313716),
"w" : NumberLong(915146120),
"W" : NumberLong(136)
},
"acquireWaitCount" : {
"r" : NumberLong(270),
"W" : NumberLong(5)
},
"timeAcquiringMicros" : {
"r" : NumberLong(1004235560),
"W" : NumberLong(756)
}
},
"MMAPV1Journal" : {
"acquireCount" : {
"r" : NumberLong(174801344),
"w" : NumberLong(955655937)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(169533721),
"w" : NumberLong(915110949),
"R" : NumberLong(16),
"W" : NumberLong(64)
},
"acquireWaitCount" : {
"r" : NumberLong(6),
"W" : NumberLong(1)
},
"timeAcquiringMicros" : {
"r" : NumberLong(36503468),
"W" : NumberLong(139)
}
},
"Collection" : {
"acquireCount" : {
"R" : NumberLong(145732116),
"W" : NumberLong(457869148)
},
"acquireWaitCount" : {
"R" : NumberLong(4695648),
"W" : NumberLong(29664317)
},
"timeAcquiringMicros" : {
"R" : NumberLong(875948378),
"W" : NumberLong("5304444496")
}
},
"Metadata" : {
"acquireCount" : {
"W" : NumberLong(224)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(5877),
"W" : NumberLong(136)
}
},
"oplog" : {
"acquireCount" : {
"R" : NumberLong(24347518),
"W" : NumberLong(457240578)
},
"acquireWaitCount" : {
"R" : NumberLong(573130),
"W" : NumberLong(9268834)
},
"timeAcquiringMicros" : {
"R" : NumberLong(62988274),
"W" : NumberLong(809457017)
}
}
},
"logicalSessionRecordCache" : {
"activeSessionsCount" : 224,
"sessionsCollectionJobCount" : 1172,
"lastSessionsCollectionJobDurationMillis" : 110,
"lastSessionsCollectionJobTimestamp" : ISODate("2020-04-11T17:17:54.398Z"),
"lastSessionsCollectionJobEntriesRefreshed" : 147,
"lastSessionsCollectionJobEntriesEnded" : 142,
"lastSessionsCollectionJobCursorsClosed" : 0,
"transactionReaperJobCount" : 1172,
"lastTransactionReaperJobDurationMillis" : 0,
"lastTransactionReaperJobTimestamp" : ISODate("2020-04-11T17:17:54.408Z"),
"lastTransactionReaperJobEntriesCleanedUp" : 0
},
"network" : {
"bytesIn" : NumberLong("326805141325"),
"bytesOut" : NumberLong("659424467058"),
"physicalBytesIn" : NumberLong("311046556853"),
"physicalBytesOut" : NumberLong("421365369268"),
"numRequests" : NumberLong(773938805),
"compression" : {
"snappy" : {
"compressor" : {
"bytesIn" : NumberLong("339255270461"),
"bytesOut" : NumberLong("100679606642")
},
"decompressor" : {
"bytesIn" : NumberLong("22214791799"),
"bytesOut" : NumberLong("40462765820")
}
}
},
"serviceExecutorTaskStats" : {
"executor" : "passthrough",
"threadsRunning" : 360
}
},
"opLatencies" : {
"reads" : {
"latency" : NumberLong("11321420023"),
"ops" : NumberLong(165395215)
},
"writes" : {
"latency" : NumberLong("72133410059"),
"ops" : NumberLong(456262434)
},
"commands" : {
"latency" : NumberLong("4920879194"),
"ops" : NumberLong(152281155)
}
},
"opcounters" : {
"insert" : 0,
"query" : 141562315,
"update" : 358012592,
"delete" : 99271652,
"getmore" : 23838703,
"command" : 152283579
},
"opcountersRepl" : {
"insert" : 118714,
"query" : 0,
"update" : 309814,
"delete" : 117900,
"getmore" : 0,
"command" : 0
},
"repl" : {
"hosts" : [
"[2606:ae00:3001:8311:172:16:244:e]:27041",
"[2606:ae00:3001:8311:172:16:244:d]:27041",
"[2606:ae00:3001:8311:172:16:244:34]:27041"
],
"setName" : "rs-shard-34",
"setVersion" : 14,
"ismaster" : true,
"secondary" : false,
"primary" : "[2606:ae00:3001:8311:172:16:244:34]:27041",
"me" : "[2606:ae00:3001:8311:172:16:244:34]:27041",
"electionId" : ObjectId("7fffffff0000000000000032"),
"lastWrite" : {
"opTime" : {
"ts" : Timestamp(1586625651, 1250),
"t" : NumberLong(50)
},
"lastWriteDate" : ISODate("2020-04-11T17:20:51Z"),
"majorityOpTime" : {
"ts" : Timestamp(1586625651, 1204),
"t" : NumberLong(50)
},
"majorityWriteDate" : ISODate("2020-04-11T17:20:51Z")
},
"rbid" : 1
},
"storageEngine" : {
"name" : "mmapv1",
"supportsCommittedReads" : false,
"readOnly" : false,
"persistent" : true
},
"tcmalloc" : {
"generic" : {
"current_allocated_bytes" : 44042192,
"heap_size" : 360931328
},
"tcmalloc" : {
"pageheap_free_bytes" : 35778560,
"pageheap_unmapped_bytes" : 203984896,
"max_total_thread_cache_bytes" : NumberLong(1073741824),
"current_total_thread_cache_bytes" : 54490096,
"total_free_bytes" : 77121776,
"central_cache_free_bytes" : 17462112,
"transfer_cache_free_bytes" : 5173472,
"thread_cache_free_bytes" : 54481328,
"aggressive_memory_decommit" : 0,
"pageheap_committed_bytes" : 156946432,
"pageheap_scavenge_count" : 1970981,
"pageheap_commit_count" : 2789214,
"pageheap_total_commit_bytes" : NumberLong("2477157507072"),
"pageheap_decommit_count" : 1970981,
"pageheap_total_decommit_bytes" : NumberLong("2477000560640"),
"pageheap_reserve_count" : 45,
"pageheap_total_reserve_bytes" : 360931328,
"spinlock_total_delay_ns" : NumberLong("67961279365"),
"formattedString" : "------------------------------------------------\nMALLOC: 44055440 ( 42.0 MiB) Bytes in use by application\nMALLOC: + 35778560 ( 34.1 MiB) Bytes in page heap freelist\nMALLOC: + 17462112 ( 16.7 MiB) Bytes in central cache freelist\nMALLOC: + 5173472 ( 4.9 MiB) Bytes in transfer cache freelist\nMALLOC: + 54476848 ( 52.0 MiB) Bytes in thread cache freelists\nMALLOC: + 3989760 ( 3.8 MiB) Bytes in malloc metadata\nMALLOC: ------------\nMALLOC: = 160936192 ( 153.5 MiB) Actual memory used (physical + swap)\nMALLOC: + 203984896 ( 194.5 MiB) Bytes released to OS (aka unmapped)\nMALLOC: ------------\nMALLOC: = 364921088 ( 348.0 MiB) Virtual address space used\nMALLOC:\nMALLOC: 10252 Spans in use\nMALLOC: 426 Thread heaps in use\nMALLOC: 4096 Tcmalloc page size\n------------------------------------------------\nCall ReleaseFreeMemory() to release freelist memory to the OS (via madvise()).\nBytes released to the OS take up virtual address space but no physical memory.\n"
}
},
"transactions" : {
"retriedCommandsCount" : NumberLong(0),
"retriedStatementsCount" : NumberLong(0),
"transactionsCollectionWriteCount" : NumberLong(0)
},
"transportSecurity" : {
"1.0" : NumberLong(0),
"1.1" : NumberLong(0),
"1.2" : NumberLong(0),
"1.3" : NumberLong(0),
"unknown" : NumberLong(0)
},
"mem" : {
"bits" : 64,
"resident" : 6057,
"virtual" : 9739,
"supported" : true,
"mapped" : 7773
},
"metrics" : {
"commands" : {
"_isSelf" : {
"failed" : NumberLong(0),
"total" : NumberLong(1)
},
"buildInfo" : {
"failed" : NumberLong(0),
"total" : NumberLong(55141)
},
"collStats" : {
"failed" : NumberLong(0),
"total" : NumberLong(49)
},
"count" : {
"failed" : NumberLong(0),
"total" : NumberLong(103)
},
"create" : {
"failed" : NumberLong(1),
"total" : NumberLong(1)
},
"createIndexes" : {
"failed" : NumberLong(0),
"total" : NumberLong(1230)
},
"delete" : {
"failed" : NumberLong(0),
"total" : NumberLong(99030933)
},
"endSessions" : {
"failed" : NumberLong(0),
"total" : NumberLong(26422)
},
"find" : {
"failed" : NumberLong(0),
"total" : NumberLong(141562316)
},
"getLastError" : {
"failed" : NumberLong(0),
"total" : NumberLong(5316)
},
"getLog" : {
"failed" : NumberLong(0),
"total" : NumberLong(14)
},
"getMore" : {
"failed" : NumberLong(0),
"total" : NumberLong(23838703)
},
"getnonce" : {
"failed" : NumberLong(0),
"total" : NumberLong(16)
},
"isMaster" : {
"failed" : NumberLong(0),
"total" : NumberLong(122572786)
},
"listCollections" : {
"failed" : NumberLong(0),
"total" : NumberLong(15)
},
"listIndexes" : {
"failed" : NumberLong(1),
"total" : NumberLong(1173)
},
"logout" : {
"failed" : NumberLong(0),
"total" : NumberLong(17)
},
"ping" : {
"failed" : NumberLong(0),
"total" : NumberLong(29)
},
"replSetGetConfig" : {
"failed" : NumberLong(0),
"total" : NumberLong(2011)
},
"replSetGetRBID" : {
"failed" : NumberLong(0),
"total" : NumberLong(2)
},
"replSetGetStatus" : {
"failed" : NumberLong(0),
"total" : NumberLong(156179)
},
"replSetHeartbeat" : {
"failed" : NumberLong(0),
"total" : NumberLong(5395247)
},
"replSetUpdatePosition" : {
"failed" : NumberLong(0),
"total" : NumberLong(23833422)
},
"saslContinue" : {
"failed" : NumberLong(0),
"total" : NumberLong(78600)
},
"saslStart" : {
"failed" : NumberLong(945),
"total" : NumberLong(40245)
},
"serverStatus" : {
"failed" : NumberLong(0),
"total" : NumberLong(91164)
},
"update" : {
"failed" : NumberLong(0),
"total" : NumberLong(357233300)
},
"whatsmyuri" : {
"failed" : NumberLong(0),
"total" : NumberLong(24396)
}
},
"cursor" : {
"timedOut" : NumberLong(0),
"open" : {
"noTimeout" : NumberLong(0),
"pinned" : NumberLong(0),
"total" : NumberLong(2)
}
},
"document" : {
"deleted" : NumberLong(99223517),
"inserted" : NumberLong(0),
"returned" : NumberLong(1056034713),
"updated" : NumberLong(258788820)
},
"getLastError" : {
"wtime" : {
"num" : 4726,
"totalMillis" : 232315
},
"wtimeouts" : NumberLong(0)
},
"operation" : {
"scanAndOrder" : NumberLong(4),
"writeConflicts" : NumberLong(0)
},
"queryExecutor" : {
"scanned" : NumberLong(499566395),
"scannedObjects" : NumberLong(1414047278)
},
"record" : {
"moves" : NumberLong(1)
},
"repl" : {
"executor" : {
"pool" : {
"inProgressCount" : 0
},
"queues" : {
"networkInProgress" : 0,
"sleepers" : 6
},
"unsignaledEvents" : 0,
"shuttingDown" : false,
"networkInterface" : "\nNetworkInterfaceASIO Operations' Diagnostic:\nOperation: Count: \nConnecting 0 \nIn Progress 0 \nSucceeded 5394020 \nCanceled 3 \nFailed 0 \nTimed Out 0 \n\n"
},
"apply" : {
"attemptsToBecomeSecondary" : NumberLong(1),
"batches" : {
"num" : 127,
"totalMillis" : 334556
},
"ops" : NumberLong(543752)
},
"buffer" : {
"count" : NumberLong(0),
"maxSizeBytes" : NumberLong(268435456),
"sizeBytes" : NumberLong(0)
},
"initialSync" : {
"completed" : NumberLong(1),
"failedAttempts" : NumberLong(0),
"failures" : NumberLong(0)
},
"network" : {
"bytes" : NumberLong(190522612),
"getmores" : {
"num" : 1156,
"totalMillis" : 61905
},
"ops" : NumberLong(564436),
"readersCreated" : NumberLong(4)
},
"preload" : {
"docs" : {
"num" : 309814,
"totalMillis" : 211
},
"indexes" : {
"num" : 1155088,
"totalMillis" : 399
}
}
},
"storage" : {
"freelist" : {
"search" : {
"bucketExhausted" : NumberLong(0),
"requests" : NumberLong(105282021),
"scanned" : NumberLong(0)
}
}
},
"ttl" : {
"deletedDocuments" : NumberLong(6702),
"passes" : NumberLong(5851)
}
},
"ok" : 1,
"operationTime" : Timestamp(1586625651, 1251),
"$clusterTime" : {
"clusterTime" : Timestamp(1586625651, 1251),
"signature" : {
"hash" : BinData(0,"Ii1/aSkK5Ad0TW0GyfF2nkebgO4="),
"keyId" : NumberLong("6806977870817132545")
}
}
}
rs-shard-34:PRIMARY>