Bug #3342
Updated by Victor Julien over 4 years ago
From a couple of days Suricata 5.0 has been crashing on a regular basis, around the same time. We finally got the core and some backtrace. Let me know what else I can do to help debug it. Backtrace <pre> (gdb) bt #0 __GI___libc_free (mem=0x7fe28bffa010) at malloc.c:3104 #1 0x00005613ebd127cd in StreamingBufferClear (sb=0x7fe5fe87c420) at util-streaming-buffer.c:392 #2 StreamingBufferClear (sb=0x7fe5fe87c420) at util-streaming-buffer.c:132 #3 0x00005613ebd127e7 in StreamingBufferFree (sb=0x7fe5fe87c420) at util-streaming-buffer.c:148 #4 0x00005613ebcd4ec4 in FileFree (ff=0x7fe5febbcc50) at util-file.c:482 #5 0x00005613ebcd643c in FileContainerRecycle (ffc=0x7fe5fe67d718) at util-file.c:406 #6 0x00005613ebd7ea1c in suricata::filecontainer::FileContainer::free (self=0x7fe5fe67d718) at src/filecontainer.rs:41 #7 suricata::smb::files::SMBFiles::free (self=<optimized out>) at src/smb/files.rs:67 #8 suricata::smb::smb::SMBState::free (self=<optimized out>) at src/smb/smb.rs:838 #9 rs_smb_state_free (state=<optimized out>) at src/smb/smb.rs:1791 #10 0x00005613ebb2ad87 in AppLayerParserStateCleanup (f=f@entry=0x7fe57214e2f0, alstate=<optimized out>, pstate=0x7fe5fe61bbf0) at app-layer-parser.c:1414 #11 0x00005613ebc06983 in FlowCleanupAppLayer (f=0x7fe57214e2f0) at flow.c:130 #12 FlowCleanupAppLayer (f=0x7fe57214e2f0) at flow.c:125 #13 0x00005613ebc08999 in FlowClearMemory (f=f@entry=0x7fe57214e2f0, proto_map=<optimized out>) at flow.c:1041 #14 0x00005613ebc0bafc in FlowRecycler (th_v=0x561401737c90, thread_data=0x7fe564000b60) at flow-manager.c:997 #15 0x00005613ebcaac0c in TmThreadsManagement (td=0x561401737c90) at tm-threads.c:706 #16 0x00007fe62a87987f in start_thread (arg=<optimized out>) at pthread_create.c:479 #17 0x00007fe629d35e03 in clone () at arena.c:290 </pre> Went up to find the flow, got the source and destination IP addresses that tell me that's some kind of SMB flow between the domain controller and servers. <pre> (gdb) frame 13 #13 0x00005613ebc08999 in FlowClearMemory (f=f@entry=0x7fe57214e2f0, proto_map=<optimized out>) at flow.c:1041 1041 flow.c: No such file or directory. (gdb) print *f $1 = {src = {address = {address_un_data32 = {489173258, 0, 0, 0}, address_un_data16 = {12554, 7464, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\n1(\035", '\000' <repeats 11 times>}}, dst = {address = {address_un_data32 = {1749364746, 0, 0, 0}, address_un_data16 = {12298, 26693, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\n0Eh", '\000' <repeats 11 times>}}, {sp = 49668, icmp_s = {type = 4 '\004', code = 194 '\302'}}, {dp = 445, icmp_d = {type = 189 '\275', code = 1 '\001'}}, proto = 6 '\006', recursion_level = 0 '\000', vlan_id = {240, 0}, vlan_idx = 1 '\001', livedev = 0x5613ed64e730, flow_hash = 3288764968, lastts = {tv_sec = 1574112362, tv_usec = 320281}, flow_state_sc_atomic__ = 2, use_cnt_sc_atomic__ = 0, tenant_id = 0, probing_parser_toserver_alproto_masks = 0, probing_parser_toclient_alproto_masks = 0, flags = 1651483, file_flags = 4092, protodetect_dp = 0, parent_id = 0, m = { __data = {__lock = 1, __count = 0, __owner = 187548, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\001\000\000\000\000\000\000\000\234\334\002\000\001", '\000' <repeats 26 times>, __align = 1}, protoctx = 0x7fe5fd8b5b70, protomap = 0 '\000', flow_end_flags = 20 '\024', alproto = 8, alproto_ts = 8, alproto_tc = 8, alproto_orig = 0, alproto_expect = 0, de_ctx_version = 104, thread_id = {2, 2}, min_ttl_toserver = 128 '\200', max_ttl_toserver = 128 '\200', min_ttl_toclient = 64 '@', max_ttl_toclient = 127 '\177', alparser = 0x7fe5fe61bbf0, alstate = 0x7fe5fe67d5c0, sgh_toclient = 0x5613fbe2ddf0, sgh_toserver = 0x5613f3e36460, flowvar = 0x0, hnext = 0x0, hprev = 0x0, fb = 0x7fe625372a40, lnext = 0x0, lprev = 0x0, startts = {tv_sec = 1574111808, tv_usec = 91686}, todstpktcnt = 397335, tosrcpktcnt = 2535584, todstbytecnt = 27513783, tosrcbytecnt = 3818073044} </pre> <pre> (gdb) frame 10 #10 0x00005613ebb2ad87 in AppLayerParserStateCleanup (f=f@entry=0x7fe57214e2f0, alstate=<optimized out>, pstate=0x7fe5fe61bbf0) at app-layer-parser.c:1414 1414 app-layer-parser.c: No such file or directory. (gdb) print *f $2 = {src = {address = {address_un_data32 = {489173258, 0, 0, 0}, address_un_data16 = {12554, 7464, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\n1(\035", '\000' <repeats 11 times>}}, dst = {address = {address_un_data32 = {1749364746, 0, 0, 0}, address_un_data16 = {12298, 26693, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\n0Eh", '\000' <repeats 11 times>}}, {sp = 49668, icmp_s = {type = 4 '\004', code = 194 '\302'}}, {dp = 445, icmp_d = {type = 189 '\275', code = 1 '\001'}}, proto = 6 '\006', recursion_level = 0 '\000', vlan_id = {240, 0}, vlan_idx = 1 '\001', livedev = 0x5613ed64e730, flow_hash = 3288764968, lastts = {tv_sec = 1574112362, tv_usec = 320281}, flow_state_sc_atomic__ = 2, use_cnt_sc_atomic__ = 0, tenant_id = 0, probing_parser_toserver_alproto_masks = 0, probing_parser_toclient_alproto_masks = 0, flags = 1651483, file_flags = 4092, protodetect_dp = 0, parent_id = 0, m = { __data = {__lock = 1, __count = 0, __owner = 187548, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\001\000\000\000\000\000\000\000\234\334\002\000\001", '\000' <repeats 26 times>, __align = 1}, protoctx = 0x7fe5fd8b5b70, protomap = 0 '\000', flow_end_flags = 20 '\024', alproto = 8, alproto_ts = 8, alproto_tc = 8, alproto_orig = 0, alproto_expect = 0, de_ctx_version = 104, thread_id = {2, 2}, min_ttl_toserver = 128 '\200', max_ttl_toserver = 128 '\200', min_ttl_toclient = 64 '@', max_ttl_toclient = 127 '\177', alparser = 0x7fe5fe61bbf0, alstate = 0x7fe5fe67d5c0, sgh_toclient = 0x5613fbe2ddf0, sgh_toserver = 0x5613f3e36460, flowvar = 0x0, hnext = 0x0, hprev = 0x0, fb = 0x7fe625372a40, lnext = 0x0, lprev = 0x0, startts = {tv_sec = 1574111808, tv_usec = 91686}, todstpktcnt = 397335, tosrcpktcnt = 2535584, todstbytecnt = 27513783, tosrcbytecnt = 3818073044} </pre> src 10.49.40.29 -> dst 10.48.69.104 so a connection was initiated from a server to the AD controller.