Invalid access at address: 0 for Mongo v4.0.5
0
I am running mongo on kubernettes in statefulsets. It was working fine for weeks then all of a sudden this morning i started getting errors where mongodb was failing.
I tried running:
mongod --repair --dbpath /database/db --storageEngine wiredTiger
but this did not seem to do anything because it would crash with the bellow logs. Any idea how to resolve this?
2019-01-02T18:05:11.621+0000 I ASIO [Replication] Dropping all pooled connections to mongod-1.mongodb-service.default.svc.cluster.local:27017 due to HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:11.621+0000 D - [Replication] User Assertion: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused src/mongo/executor/network_interface_tl.cpp 250
2019-01-02T18:05:11.621+0000 I REPL_HB [replexec-0] Error in heartbeat (requestId: 176) to mongod-1.mongodb-service.default.svc.cluster.local:27017, response status: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:12.087+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory Raw: [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory Raw: [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory Raw: [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
2019-01-02T18:05:12.088+0000 F - [thread7] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 409
2019-01-02T18:05:12.088+0000 F - [thread7]
***aborting after fassert() failure
2019-01-02T18:05:12.099+0000 F - [thread7] Got signal: 6 (Aborted).
0x55a040cf22e1 0x55a040cf14f9 0x55a040cf19dd 0x7faf48d8e390 0x7faf489e8428 0x7faf489ea02a 0x55a03f31aa13 0x55a03f411506 0x55a03f482e59 0x55a03f2a4f1b 0x55a03f2a533b 0x55a03f4ed692 0x7faf48d846ba 0x7faf48aba41d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55A03E907000","o":"23EB2E1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55A03E907000","o":"23EA4F9"},{"b":"55A03E907000","o":"23EA9DD"},{"b":"7FAF48D7D000","o":"11390"},{"b":"7FAF489B3000","o":"35428","s":"gsignal"},{"b":"7FAF489B3000","o":"3702A","s":"abort"},{"b":"55A03E907000","o":"A13A13","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55A03E907000","o":"B0A506"},{"b":"55A03E907000","o":"B7BE59"},{"b":"55A03E907000","o":"99DF1B","s":"__wt_err_func"},{"b":"55A03E907000","o":"99E33B","s":"__wt_panic"},{"b":"55A03E907000","o":"BE6692"},{"b":"7FAF48D7D000","o":"76BA"},{"b":"7FAF489B3000","o":"10741D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.5", "gitVersion" : "3739429dd92b92d1b0ab120911a23d50bf03c412", "compiledModules" : , "uname" : { "sysname" : "Linux", "release" : "4.15.0-1030-azure", "version" : "#31~16.04.1-Ubuntu SMP Tue Oct 30 19:40:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55A03E907000", "elfType" : 3, "buildId" : "D2BD849936E6F1CDEAA6678F57E07D725B522378" }, { "b" : "7FFC58578000", "elfType" : 3, "buildId" : "EB4266DE01729239D0A2A1F10C5AF3A733C81B52" }, { "b" : "7FAF4A18E000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "E2D1DB1857AE8F07908959381751796E89FF56BB" }, { "b" : "7FAF49F73000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }, { "b" : "7FAF49B2E000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "8942CA58A3B910E883CC31E04A23DBD09729B4B0" }, { "b" : "7FAF498C5000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "473092A9AF373FB0CAB555F9A003BC67F47756B6" }, { "b" : "7FAF496C1000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }, { "b" : "7FAF494B9000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3" }, { "b" : "7FAF491B0000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }, { "b" : "7FAF48F9A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FAF48D7D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493D30" }, { "b" : "7FAF489B3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }, { "b" : "7FAF4A3FD000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }, { "b" : "7FAF48780000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, { "b" : "7FAF48564000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, { "b" : "7FAF4831A000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DB5180B568097E2A4690A5B40D36BD134C893FEE" }, { "b" : "7FAF4810B000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5B146086EC173C299BA45F0C92E068D68CC7AD37" }, { "b" : "7FAF47EBA000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "D02A94DB90567C2C45F40C8565809A041FD22A55" }, { "b" : "7FAF47CA0000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, { "b" : "7FAF47970000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, { "b" : "7FAF4773D000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, { "b" : "7FAF47507000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, { "b" : "7FAF47287000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, { "b" : "7FAF46FB5000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "16E3DBC6D048145939BB43BBFD7954D27421B00F" }, { "b" : "7FAF46D86000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "AEB4C08FC47F86C475E9D3996DFE5E9B403ACEBF" }, { "b" : "7FAF46B82000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, { "b" : "7FAF46977000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "DF3219B89E86920E901BAC4A80AA60F2B6134588" }, { "b" : "7FAF4675C000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "87783DF8A1058CD150F8886CB36340384093C18F" }, { "b" : "7FAF4651B000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, { "b" : "7FAF462B7000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, { "b" : "7FAF460A4000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, { "b" : "7FAF45EA0000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, { "b" : "7FAF45C97000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, { "b" : "7FAF45A0D000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, { "b" : "7FAF4576B000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, { "b" : "7FAF45538000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, { "b" : "7FAF45322000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, { "b" : "7FAF4511A000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, { "b" : "7FAF44EF1000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, { "b" : "7FAF44CE2000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, { "b" : "7FAF44A97000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, { "b" : "7FAF447C2000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "D9782BA023CAEC26B15D8676E3A5D07B55E121EF" }, { "b" : "7FAF4458A000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "7BDD51353D50310FFA1587E4AA01B40ABE32D582" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55a040cf22e1]
mongod(+0x23EA4F9) [0x55a040cf14f9]
mongod(+0x23EA9DD) [0x55a040cf19dd]
libpthread.so.0(+0x11390) [0x7faf48d8e390]
libc.so.6(gsignal+0x38) [0x7faf489e8428]
libc.so.6(abort+0x16A) [0x7faf489ea02a]
mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55a03f31aa13]
mongod(+0xB0A506) [0x55a03f411506]
mongod(+0xB7BE59) [0x55a03f482e59]
mongod(__wt_err_func+0x90) [0x55a03f2a4f1b]
mongod(__wt_panic+0x3F) [0x55a03f2a533b]
mongod(+0xBE6692) [0x55a03f4ed692]
libpthread.so.0(+0x76BA) [0x7faf48d846ba]
libc.so.6(clone+0x6D) [0x7faf48aba41d]
----- END BACKTRACE -----
mongodb mongodb-4.0
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
migrated from superuser.com Jan 2 at 19:45
This question came from our site for computer enthusiasts and power users.
add a comment |
0
I am running mongo on kubernettes in statefulsets. It was working fine for weeks then all of a sudden this morning i started getting errors where mongodb was failing.
I tried running:
mongod --repair --dbpath /database/db --storageEngine wiredTiger
but this did not seem to do anything because it would crash with the bellow logs. Any idea how to resolve this?
2019-01-02T18:05:11.621+0000 I ASIO [Replication] Dropping all pooled connections to mongod-1.mongodb-service.default.svc.cluster.local:27017 due to HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:11.621+0000 D - [Replication] User Assertion: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused src/mongo/executor/network_interface_tl.cpp 250
2019-01-02T18:05:11.621+0000 I REPL_HB [replexec-0] Error in heartbeat (requestId: 176) to mongod-1.mongodb-service.default.svc.cluster.local:27017, response status: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:12.087+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory Raw: [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory Raw: [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory Raw: [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
2019-01-02T18:05:12.088+0000 F - [thread7] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 409
2019-01-02T18:05:12.088+0000 F - [thread7]
***aborting after fassert() failure
2019-01-02T18:05:12.099+0000 F - [thread7] Got signal: 6 (Aborted).
0x55a040cf22e1 0x55a040cf14f9 0x55a040cf19dd 0x7faf48d8e390 0x7faf489e8428 0x7faf489ea02a 0x55a03f31aa13 0x55a03f411506 0x55a03f482e59 0x55a03f2a4f1b 0x55a03f2a533b 0x55a03f4ed692 0x7faf48d846ba 0x7faf48aba41d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55A03E907000","o":"23EB2E1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55A03E907000","o":"23EA4F9"},{"b":"55A03E907000","o":"23EA9DD"},{"b":"7FAF48D7D000","o":"11390"},{"b":"7FAF489B3000","o":"35428","s":"gsignal"},{"b":"7FAF489B3000","o":"3702A","s":"abort"},{"b":"55A03E907000","o":"A13A13","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55A03E907000","o":"B0A506"},{"b":"55A03E907000","o":"B7BE59"},{"b":"55A03E907000","o":"99DF1B","s":"__wt_err_func"},{"b":"55A03E907000","o":"99E33B","s":"__wt_panic"},{"b":"55A03E907000","o":"BE6692"},{"b":"7FAF48D7D000","o":"76BA"},{"b":"7FAF489B3000","o":"10741D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.5", "gitVersion" : "3739429dd92b92d1b0ab120911a23d50bf03c412", "compiledModules" : , "uname" : { "sysname" : "Linux", "release" : "4.15.0-1030-azure", "version" : "#31~16.04.1-Ubuntu SMP Tue Oct 30 19:40:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55A03E907000", "elfType" : 3, "buildId" : "D2BD849936E6F1CDEAA6678F57E07D725B522378" }, { "b" : "7FFC58578000", "elfType" : 3, "buildId" : "EB4266DE01729239D0A2A1F10C5AF3A733C81B52" }, { "b" : "7FAF4A18E000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "E2D1DB1857AE8F07908959381751796E89FF56BB" }, { "b" : "7FAF49F73000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }, { "b" : "7FAF49B2E000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "8942CA58A3B910E883CC31E04A23DBD09729B4B0" }, { "b" : "7FAF498C5000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "473092A9AF373FB0CAB555F9A003BC67F47756B6" }, { "b" : "7FAF496C1000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }, { "b" : "7FAF494B9000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3" }, { "b" : "7FAF491B0000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }, { "b" : "7FAF48F9A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FAF48D7D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493D30" }, { "b" : "7FAF489B3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }, { "b" : "7FAF4A3FD000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }, { "b" : "7FAF48780000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, { "b" : "7FAF48564000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, { "b" : "7FAF4831A000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DB5180B568097E2A4690A5B40D36BD134C893FEE" }, { "b" : "7FAF4810B000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5B146086EC173C299BA45F0C92E068D68CC7AD37" }, { "b" : "7FAF47EBA000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "D02A94DB90567C2C45F40C8565809A041FD22A55" }, { "b" : "7FAF47CA0000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, { "b" : "7FAF47970000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, { "b" : "7FAF4773D000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, { "b" : "7FAF47507000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, { "b" : "7FAF47287000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, { "b" : "7FAF46FB5000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "16E3DBC6D048145939BB43BBFD7954D27421B00F" }, { "b" : "7FAF46D86000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "AEB4C08FC47F86C475E9D3996DFE5E9B403ACEBF" }, { "b" : "7FAF46B82000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, { "b" : "7FAF46977000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "DF3219B89E86920E901BAC4A80AA60F2B6134588" }, { "b" : "7FAF4675C000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "87783DF8A1058CD150F8886CB36340384093C18F" }, { "b" : "7FAF4651B000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, { "b" : "7FAF462B7000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, { "b" : "7FAF460A4000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, { "b" : "7FAF45EA0000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, { "b" : "7FAF45C97000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, { "b" : "7FAF45A0D000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, { "b" : "7FAF4576B000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, { "b" : "7FAF45538000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, { "b" : "7FAF45322000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, { "b" : "7FAF4511A000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, { "b" : "7FAF44EF1000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, { "b" : "7FAF44CE2000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, { "b" : "7FAF44A97000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, { "b" : "7FAF447C2000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "D9782BA023CAEC26B15D8676E3A5D07B55E121EF" }, { "b" : "7FAF4458A000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "7BDD51353D50310FFA1587E4AA01B40ABE32D582" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55a040cf22e1]
mongod(+0x23EA4F9) [0x55a040cf14f9]
mongod(+0x23EA9DD) [0x55a040cf19dd]
libpthread.so.0(+0x11390) [0x7faf48d8e390]
libc.so.6(gsignal+0x38) [0x7faf489e8428]
libc.so.6(abort+0x16A) [0x7faf489ea02a]
mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55a03f31aa13]
mongod(+0xB0A506) [0x55a03f411506]
mongod(+0xB7BE59) [0x55a03f482e59]
mongod(__wt_err_func+0x90) [0x55a03f2a4f1b]
mongod(__wt_panic+0x3F) [0x55a03f2a533b]
mongod(+0xBE6692) [0x55a03f4ed692]
libpthread.so.0(+0x76BA) [0x7faf48d846ba]
libc.so.6(clone+0x6D) [0x7faf48aba41d]
----- END BACKTRACE -----
mongodb mongodb-4.0
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
migrated from superuser.com Jan 2 at 19:45
This question came from our site for computer enthusiasts and power users.
add a comment |
0
0
0
I am running mongo on kubernettes in statefulsets. It was working fine for weeks then all of a sudden this morning i started getting errors where mongodb was failing.
I tried running:
mongod --repair --dbpath /database/db --storageEngine wiredTiger
but this did not seem to do anything because it would crash with the bellow logs. Any idea how to resolve this?
2019-01-02T18:05:11.621+0000 I ASIO [Replication] Dropping all pooled connections to mongod-1.mongodb-service.default.svc.cluster.local:27017 due to HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:11.621+0000 D - [Replication] User Assertion: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused src/mongo/executor/network_interface_tl.cpp 250
2019-01-02T18:05:11.621+0000 I REPL_HB [replexec-0] Error in heartbeat (requestId: 176) to mongod-1.mongodb-service.default.svc.cluster.local:27017, response status: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:12.087+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory Raw: [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory Raw: [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory Raw: [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
2019-01-02T18:05:12.088+0000 F - [thread7] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 409
2019-01-02T18:05:12.088+0000 F - [thread7]
***aborting after fassert() failure
2019-01-02T18:05:12.099+0000 F - [thread7] Got signal: 6 (Aborted).
0x55a040cf22e1 0x55a040cf14f9 0x55a040cf19dd 0x7faf48d8e390 0x7faf489e8428 0x7faf489ea02a 0x55a03f31aa13 0x55a03f411506 0x55a03f482e59 0x55a03f2a4f1b 0x55a03f2a533b 0x55a03f4ed692 0x7faf48d846ba 0x7faf48aba41d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55A03E907000","o":"23EB2E1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55A03E907000","o":"23EA4F9"},{"b":"55A03E907000","o":"23EA9DD"},{"b":"7FAF48D7D000","o":"11390"},{"b":"7FAF489B3000","o":"35428","s":"gsignal"},{"b":"7FAF489B3000","o":"3702A","s":"abort"},{"b":"55A03E907000","o":"A13A13","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55A03E907000","o":"B0A506"},{"b":"55A03E907000","o":"B7BE59"},{"b":"55A03E907000","o":"99DF1B","s":"__wt_err_func"},{"b":"55A03E907000","o":"99E33B","s":"__wt_panic"},{"b":"55A03E907000","o":"BE6692"},{"b":"7FAF48D7D000","o":"76BA"},{"b":"7FAF489B3000","o":"10741D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.5", "gitVersion" : "3739429dd92b92d1b0ab120911a23d50bf03c412", "compiledModules" : , "uname" : { "sysname" : "Linux", "release" : "4.15.0-1030-azure", "version" : "#31~16.04.1-Ubuntu SMP Tue Oct 30 19:40:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55A03E907000", "elfType" : 3, "buildId" : "D2BD849936E6F1CDEAA6678F57E07D725B522378" }, { "b" : "7FFC58578000", "elfType" : 3, "buildId" : "EB4266DE01729239D0A2A1F10C5AF3A733C81B52" }, { "b" : "7FAF4A18E000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "E2D1DB1857AE8F07908959381751796E89FF56BB" }, { "b" : "7FAF49F73000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }, { "b" : "7FAF49B2E000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "8942CA58A3B910E883CC31E04A23DBD09729B4B0" }, { "b" : "7FAF498C5000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "473092A9AF373FB0CAB555F9A003BC67F47756B6" }, { "b" : "7FAF496C1000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }, { "b" : "7FAF494B9000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3" }, { "b" : "7FAF491B0000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }, { "b" : "7FAF48F9A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FAF48D7D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493D30" }, { "b" : "7FAF489B3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }, { "b" : "7FAF4A3FD000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }, { "b" : "7FAF48780000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, { "b" : "7FAF48564000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, { "b" : "7FAF4831A000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DB5180B568097E2A4690A5B40D36BD134C893FEE" }, { "b" : "7FAF4810B000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5B146086EC173C299BA45F0C92E068D68CC7AD37" }, { "b" : "7FAF47EBA000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "D02A94DB90567C2C45F40C8565809A041FD22A55" }, { "b" : "7FAF47CA0000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, { "b" : "7FAF47970000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, { "b" : "7FAF4773D000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, { "b" : "7FAF47507000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, { "b" : "7FAF47287000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, { "b" : "7FAF46FB5000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "16E3DBC6D048145939BB43BBFD7954D27421B00F" }, { "b" : "7FAF46D86000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "AEB4C08FC47F86C475E9D3996DFE5E9B403ACEBF" }, { "b" : "7FAF46B82000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, { "b" : "7FAF46977000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "DF3219B89E86920E901BAC4A80AA60F2B6134588" }, { "b" : "7FAF4675C000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "87783DF8A1058CD150F8886CB36340384093C18F" }, { "b" : "7FAF4651B000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, { "b" : "7FAF462B7000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, { "b" : "7FAF460A4000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, { "b" : "7FAF45EA0000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, { "b" : "7FAF45C97000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, { "b" : "7FAF45A0D000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, { "b" : "7FAF4576B000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, { "b" : "7FAF45538000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, { "b" : "7FAF45322000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, { "b" : "7FAF4511A000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, { "b" : "7FAF44EF1000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, { "b" : "7FAF44CE2000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, { "b" : "7FAF44A97000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, { "b" : "7FAF447C2000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "D9782BA023CAEC26B15D8676E3A5D07B55E121EF" }, { "b" : "7FAF4458A000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "7BDD51353D50310FFA1587E4AA01B40ABE32D582" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55a040cf22e1]
mongod(+0x23EA4F9) [0x55a040cf14f9]
mongod(+0x23EA9DD) [0x55a040cf19dd]
libpthread.so.0(+0x11390) [0x7faf48d8e390]
libc.so.6(gsignal+0x38) [0x7faf489e8428]
libc.so.6(abort+0x16A) [0x7faf489ea02a]
mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55a03f31aa13]
mongod(+0xB0A506) [0x55a03f411506]
mongod(+0xB7BE59) [0x55a03f482e59]
mongod(__wt_err_func+0x90) [0x55a03f2a4f1b]
mongod(__wt_panic+0x3F) [0x55a03f2a533b]
mongod(+0xBE6692) [0x55a03f4ed692]
libpthread.so.0(+0x76BA) [0x7faf48d846ba]
libc.so.6(clone+0x6D) [0x7faf48aba41d]
----- END BACKTRACE -----
mongodb mongodb-4.0
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
I am running mongo on kubernettes in statefulsets. It was working fine for weeks then all of a sudden this morning i started getting errors where mongodb was failing.
I tried running:
mongod --repair --dbpath /database/db --storageEngine wiredTiger
but this did not seem to do anything because it would crash with the bellow logs. Any idea how to resolve this?
2019-01-02T18:05:11.621+0000 I ASIO [Replication] Dropping all pooled connections to mongod-1.mongodb-service.default.svc.cluster.local:27017 due to HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:11.621+0000 D - [Replication] User Assertion: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused src/mongo/executor/network_interface_tl.cpp 250
2019-01-02T18:05:11.621+0000 I REPL_HB [replexec-0] Error in heartbeat (requestId: 176) to mongod-1.mongodb-service.default.svc.cluster.local:27017, response status: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:12.087+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory Raw: [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory Raw: [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory Raw: [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
2019-01-02T18:05:12.088+0000 F - [thread7] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 409
2019-01-02T18:05:12.088+0000 F - [thread7]
***aborting after fassert() failure
2019-01-02T18:05:12.099+0000 F - [thread7] Got signal: 6 (Aborted).
0x55a040cf22e1 0x55a040cf14f9 0x55a040cf19dd 0x7faf48d8e390 0x7faf489e8428 0x7faf489ea02a 0x55a03f31aa13 0x55a03f411506 0x55a03f482e59 0x55a03f2a4f1b 0x55a03f2a533b 0x55a03f4ed692 0x7faf48d846ba 0x7faf48aba41d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55A03E907000","o":"23EB2E1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55A03E907000","o":"23EA4F9"},{"b":"55A03E907000","o":"23EA9DD"},{"b":"7FAF48D7D000","o":"11390"},{"b":"7FAF489B3000","o":"35428","s":"gsignal"},{"b":"7FAF489B3000","o":"3702A","s":"abort"},{"b":"55A03E907000","o":"A13A13","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55A03E907000","o":"B0A506"},{"b":"55A03E907000","o":"B7BE59"},{"b":"55A03E907000","o":"99DF1B","s":"__wt_err_func"},{"b":"55A03E907000","o":"99E33B","s":"__wt_panic"},{"b":"55A03E907000","o":"BE6692"},{"b":"7FAF48D7D000","o":"76BA"},{"b":"7FAF489B3000","o":"10741D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.5", "gitVersion" : "3739429dd92b92d1b0ab120911a23d50bf03c412", "compiledModules" : , "uname" : { "sysname" : "Linux", "release" : "4.15.0-1030-azure", "version" : "#31~16.04.1-Ubuntu SMP Tue Oct 30 19:40:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55A03E907000", "elfType" : 3, "buildId" : "D2BD849936E6F1CDEAA6678F57E07D725B522378" }, { "b" : "7FFC58578000", "elfType" : 3, "buildId" : "EB4266DE01729239D0A2A1F10C5AF3A733C81B52" }, { "b" : "7FAF4A18E000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "E2D1DB1857AE8F07908959381751796E89FF56BB" }, { "b" : "7FAF49F73000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }, { "b" : "7FAF49B2E000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "8942CA58A3B910E883CC31E04A23DBD09729B4B0" }, { "b" : "7FAF498C5000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "473092A9AF373FB0CAB555F9A003BC67F47756B6" }, { "b" : "7FAF496C1000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }, { "b" : "7FAF494B9000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3" }, { "b" : "7FAF491B0000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }, { "b" : "7FAF48F9A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FAF48D7D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493D30" }, { "b" : "7FAF489B3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }, { "b" : "7FAF4A3FD000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }, { "b" : "7FAF48780000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, { "b" : "7FAF48564000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, { "b" : "7FAF4831A000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DB5180B568097E2A4690A5B40D36BD134C893FEE" }, { "b" : "7FAF4810B000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5B146086EC173C299BA45F0C92E068D68CC7AD37" }, { "b" : "7FAF47EBA000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "D02A94DB90567C2C45F40C8565809A041FD22A55" }, { "b" : "7FAF47CA0000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, { "b" : "7FAF47970000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, { "b" : "7FAF4773D000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, { "b" : "7FAF47507000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, { "b" : "7FAF47287000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, { "b" : "7FAF46FB5000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "16E3DBC6D048145939BB43BBFD7954D27421B00F" }, { "b" : "7FAF46D86000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "AEB4C08FC47F86C475E9D3996DFE5E9B403ACEBF" }, { "b" : "7FAF46B82000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, { "b" : "7FAF46977000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "DF3219B89E86920E901BAC4A80AA60F2B6134588" }, { "b" : "7FAF4675C000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "87783DF8A1058CD150F8886CB36340384093C18F" }, { "b" : "7FAF4651B000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, { "b" : "7FAF462B7000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, { "b" : "7FAF460A4000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, { "b" : "7FAF45EA0000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, { "b" : "7FAF45C97000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, { "b" : "7FAF45A0D000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, { "b" : "7FAF4576B000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, { "b" : "7FAF45538000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, { "b" : "7FAF45322000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, { "b" : "7FAF4511A000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, { "b" : "7FAF44EF1000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, { "b" : "7FAF44CE2000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, { "b" : "7FAF44A97000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, { "b" : "7FAF447C2000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "D9782BA023CAEC26B15D8676E3A5D07B55E121EF" }, { "b" : "7FAF4458A000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "7BDD51353D50310FFA1587E4AA01B40ABE32D582" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55a040cf22e1]
mongod(+0x23EA4F9) [0x55a040cf14f9]
mongod(+0x23EA9DD) [0x55a040cf19dd]
libpthread.so.0(+0x11390) [0x7faf48d8e390]
libc.so.6(gsignal+0x38) [0x7faf489e8428]
libc.so.6(abort+0x16A) [0x7faf489ea02a]
mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55a03f31aa13]
mongod(+0xB0A506) [0x55a03f411506]
mongod(+0xB7BE59) [0x55a03f482e59]
mongod(__wt_err_func+0x90) [0x55a03f2a4f1b]
mongod(__wt_panic+0x3F) [0x55a03f2a533b]
mongod(+0xBE6692) [0x55a03f4ed692]
libpthread.so.0(+0x76BA) [0x7faf48d846ba]
libc.so.6(clone+0x6D) [0x7faf48aba41d]
----- END BACKTRACE -----
mongodb mongodb-4.0
mongodb mongodb-4.0
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
edited Jan 4 at 18:25
Md Haidar Ali Khan
3,62662342
3,62662342
asked Jan 2 at 18:18
Shawn Varughese
migrated from superuser.com Jan 2 at 19:45
This question came from our site for computer enthusiasts and power users.
migrated from superuser.com Jan 2 at 19:45
This question came from our site for computer enthusiasts and power users.
add a comment |
add a comment |
1 Answer
1
active
oldest
votes
0
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error
(-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic,
523: the process must exit and restart: WT_PANIC: WiredTiger library
panic Raw: [1546452312:88097][1:0x7faf42585700], log-server:
__wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
I have gone through your mention error, as per MongoDB jira Blog here That error means WiredTiger (the storage engine for MongoDB) detected that there was a disk corruption affecting the database. Is it possible that your underlying storage system is unreliable? I recommend running integrity checks on your hardware to look for problems.
When MongoDB detects an on-disk data corruption, it exits immediately, since attempting to continue could lead to more extensive data corruption and/or data loss. The behavior you report is expected in this situation.
For further your ref here
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
add a comment |
Your Answer
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "182"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: false,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: null,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fdba.stackexchange.com%2fquestions%2f226192%2finvalid-access-at-address-0-for-mongo-v4-0-5%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
0
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error
(-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic,
523: the process must exit and restart: WT_PANIC: WiredTiger library
panic Raw: [1546452312:88097][1:0x7faf42585700], log-server:
__wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
I have gone through your mention error, as per MongoDB jira Blog here That error means WiredTiger (the storage engine for MongoDB) detected that there was a disk corruption affecting the database. Is it possible that your underlying storage system is unreliable? I recommend running integrity checks on your hardware to look for problems.
When MongoDB detects an on-disk data corruption, it exits immediately, since attempting to continue could lead to more extensive data corruption and/or data loss. The behavior you report is expected in this situation.
For further your ref here
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
add a comment |
0
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error
(-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic,
523: the process must exit and restart: WT_PANIC: WiredTiger library
panic Raw: [1546452312:88097][1:0x7faf42585700], log-server:
__wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
I have gone through your mention error, as per MongoDB jira Blog here That error means WiredTiger (the storage engine for MongoDB) detected that there was a disk corruption affecting the database. Is it possible that your underlying storage system is unreliable? I recommend running integrity checks on your hardware to look for problems.
When MongoDB detects an on-disk data corruption, it exits immediately, since attempting to continue could lead to more extensive data corruption and/or data loss. The behavior you report is expected in this situation.
For further your ref here
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
add a comment |
0
0
0
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error
(-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic,
523: the process must exit and restart: WT_PANIC: WiredTiger library
panic Raw: [1546452312:88097][1:0x7faf42585700], log-server:
__wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
I have gone through your mention error, as per MongoDB jira Blog here That error means WiredTiger (the storage engine for MongoDB) detected that there was a disk corruption affecting the database. Is it possible that your underlying storage system is unreliable? I recommend running integrity checks on your hardware to look for problems.
When MongoDB detects an on-disk data corruption, it exits immediately, since attempting to continue could lead to more extensive data corruption and/or data loss. The behavior you report is expected in this situation.
For further your ref here
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error
(-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic,
523: the process must exit and restart: WT_PANIC: WiredTiger library
panic Raw: [1546452312:88097][1:0x7faf42585700], log-server:
__wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
I have gone through your mention error, as per MongoDB jira Blog here That error means WiredTiger (the storage engine for MongoDB) detected that there was a disk corruption affecting the database. Is it possible that your underlying storage system is unreliable? I recommend running integrity checks on your hardware to look for problems.
When MongoDB detects an on-disk data corruption, it exits immediately, since attempting to continue could lead to more extensive data corruption and/or data loss. The behavior you report is expected in this situation.
For further your ref here
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
answered Jan 4 at 18:31
Md Haidar Ali KhanMd Haidar Ali Khan
3,62662342
3,62662342
add a comment |
add a comment |
draft saved
draft discarded
Thanks for contributing an answer to Database Administrators Stack Exchange!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fdba.stackexchange.com%2fquestions%2f226192%2finvalid-access-at-address-0-for-mongo-v4-0-5%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
