Project

General

Profile

Bug #4532

Updated by Victor Julien over 2 years ago

Sigsegv with Suricata 6.0.2 at a large deployment site. The following details are from the thread on which the fault occurred. 

 The deployment site sees 40-60Gbps of mixed traffic with lots of east/west and north/south traffic. 

 Stack trace 

 <pre><code class="c"> 
 #0    htp_tx_destroy_incomplete (tx=0x7f81edf5d480) at htp_transaction.c:146 
 #1    0x00007f9175eb5739 in htp_conn_destroy (conn=0x7f8219ba4810) at htp_connection.c:84 
 #2    0x00007f9175eb5ab2 in htp_connp_destroy_all (connp=0x7f81614a8b00) at htp_connection_parser.c:135 
 #3    0x000055cdf5485fea in HTPStateFree (state=0x7f81ede90bc0) at app-layer-htp.c:414 
 #4    0x000055cdf548ea00 in AppLayerParserStateProtoCleanup (pstate=0x7f81612ee740, alstate=<optimized out>, alproto=<optimized out>, protomap=<optimized out>) at app-layer-parser.c:1488 
 #5    AppLayerParserStateCleanup (f=f@entry=0x7f8155932d80, alstate=<optimized out>, pstate=0x7f81612ee740) at app-layer-parser.c:1499 
 #6    0x000055cdf5523796 in FlowCleanupAppLayer (f=0x7f8155932d80) at flow.c:147 
 #7    FlowClearMemory (f=f@entry=0x7f8155932d80, proto_map=<optimized out>) at flow.c:1075 
 #8    0x000055cdf552904d in CheckWorkQueue (tv=tv@entry=0x7f916e6e6bc0, fw=fw@entry=0x7f9105891000, detect_thread=detect_thread@entry=0x7f9104f25000, counters=counters@entry=0x7f91078fb2a8, fq=fq@entry=0x7f91078fb2b0) at flow-worker.c:201 
 #9    0x000055cdf55295e2 in FlowWorkerProcessInjectedFlows (p=0x7f9105873600, detect_thread=0x7f9104f25000, fw=0x7f9105891000, tv=0x7f916e6e6bc0) at flow-worker.c:447 
 #10 FlowWorker (tv=0x7f916e6e6bc0, p=0x7f9105873600, data=0x7f9105891000) at flow-worker.c:570 
 #11 0x000055cdf558127b in TmThreadsSlotVarRun (tv=0x7f916e6e6bc0, p=0x7f9105873600, slot=<optimized out>) at tm-threads.c:127 
 #12 0x000055cdf5563465 in TmThreadsSlotProcessPkt (p=0x7f9105873600, s=<optimized out>, tv=0x7f916e6e6bc0) at tm-threads.h:192 
 #13 NapatechPacketLoop (tv=0x7f916e6e6bc0, data=0x7f910697d000, slot=<optimized out>) at source-napatech.c:1069 
 #14 0x000055cdf5583447 in TmThreadsSlotPktAcqLoop (td=0x7f916e6e6bc0) at tm-threads.c:322 
 #15 0x00007f917498f37e in start_thread (arg=0x7f91078ff640) at pthread_create.c:463 
 #16 0x00007f9172e15b5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 

 </code></pre> 

 Thread 
 <pre><code class="c"> 
 * 1      Thread 0x7f91078ff640 (LWP 91553) htp_tx_destroy_incomplete (tx=0x7f81edf5d480) at htp_transaction.c:146 

 </code></pre> 

 Thread info 
 <pre><code class="c"> 
 (gdb) fr 11 
 #11 0x000055cdf558127b in TmThreadsSlotVarRun (tv=0x7f916e6e6bc0, p=0x7f9105873600, slot=<optimized out>) at tm-threads.c:127 
 127 	 tm-threads.c: No such file or directory. 
 (gdb) p *tv 
 $6 = {t = 140260873860672, tm_func = 0x55cdf5583210 <TmThreadsSlotPktAcqLoop>, name = "W#01-nt31\000\000\000\000\000\000", printable_name = 0x7f91710a2a60 "W#01-nt31", thread_group_name = 0x0, thread_setup_flags = 4 '\004', type = 0 '\000', cpu_affinity = 1, thread_priority = -2, 
   tmm_flags = 15 '\017', cap_flags = 32 ' ', inq_id = 2 '\002', outq_id = 2 '\002', id = 32, inq = 0x0, tmqh_in = 0x55cdf55802f0 <TmqhInputPacketpool>, flags_sc_atomic__ = 3, tm_slots = 0x7f91710a6780, tm_flowworker = 0x7f91710a6800, outq = 0x0, outctx = 0x0, 
   tmqh_out = 0x55cdf557fb10 <TmqhOutputPacketpool>, decode_pq = {top = 0x0, bot = 0x0, len = 0}, stream_pq = 0x7f91058a60e0, stream_pq_local = 0x7f91058a60e0, perf_private_ctx = {head = 0x7f9104f77000, size = 246, initialized = 1}, next = 0x7f916e6e6d00, perf_public_ctx = { 
     perf_flag = 0, head = 0x7f91062fc000, curr_id = 246, m = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}, ctrl_mutex = 0x0, 
   ctrl_cond = 0x0, flow_queue = 0x7f9104f72000} 

 </code></pre> 

 Frame 0 
 <pre><code class="c"> 
 (gdb) p *tx 
 $5 = {connp = 0x7f81614a8b00, conn = 0x7f8219ba4810, cfg = 0x7f917105b000, is_config_shared = 1, user_data = 0x0, request_ignored_lines = 0, request_line = 0x7f81edc61b30, request_method = 0x7f81ede45ec0, request_method_number = HTP_M_GET, request_uri = 0x7f821982be00, 
   request_protocol = 0x7f81ede45ee0, request_protocol_number = 101, is_protocol_0_9 = 0, parsed_uri = 0x7f81edc61b80, parsed_uri_raw = 0x7f81edc61ae0, request_message_len = 0, request_entity_len = 0, request_headers = 0x7f81b944b790, request_transfer_coding = HTP_CODING_NO_BODY, 
   request_content_encoding = HTP_COMPRESSION_UNKNOWN, request_content_type = 0x0, request_content_length = -1, hook_request_body_data = 0x0, hook_response_body_data = 0x0, request_urlenp_query = 0x0, request_urlenp_body = 0x0, request_mpartp = 0x0, request_params = 0x7f81b944b7c0, 
   request_cookies = 0x0, request_auth_type = HTP_AUTH_NONE, request_auth_username = 0x0, request_auth_password = 0x0, request_hostname = 0x7f81b944bd00, request_port_number = -1, response_ignored_lines = 0, response_line = 0x0, response_protocol = 0x0, 
   response_protocol_number = -1, response_status = 0x0, response_status_number = 0, response_status_expected_number = 0, response_message = 0x0, seen_100continue = 0, response_headers = 0x7f81b944b7f0, response_message_len = 0, response_entity_len = 0, 
   response_content_length = -1, response_transfer_coding = HTP_CODING_UNKNOWN, response_content_encoding = HTP_COMPRESSION_UNKNOWN, response_content_encoding_processing = HTP_COMPRESSION_UNKNOWN, response_content_type = 0x0, flags = 2048, request_progress = HTP_REQUEST_COMPLETE, 
   response_progress = HTP_RESPONSE_NOT_STARTED, index = 41, req_header_repetitions = 0, res_header_repetitions = 0}j 
 (gdb) p tx->request_headers 
 $3 = (htp_table_t *) 0x7f81b944b790 
 (gdb) p *tx->request_headers 
 $4 = {list = {first = 0, last = 14, max_size = 64, current_size = 14, elements = 0x7f7f391a2000}, alloc_type = HTP_TABLE_KEYS_COPIED} 
 (gdb) x tx->request_headers.list.elements@14 
 0x7f81b944b7b0: 	 0x391a2000 
 (gdb) x/14 tx->request_headers.list.elements 
 0x7f7f391a2000: 	 0x00000000 	 0x00000000 	 0x00000000 	 0x00000000 
 0x7f7f391a2010: 	 0x00000000 	 0x00000000 	 0x00000000 	 0x00000000 
 0x7f7f391a2020: 	 0x00000000 	 0x00000000 	 0x00000000 	 0x00000000 
 0x7f7f391a2030: 	 0x00000000 	 0x00000000 

 </code></pre> 

 Frame 1 
 <pre><code class="c"> 
 (gdb) fr 1 
 #1    0x00007f9175eb5739 in htp_conn_destroy (conn=0x7f8219ba4810) at htp_connection.c:84 
 84 	 htp_connection.c: No such file or directory. 
 (gdb) p *conn 
 $1 = {client_addr = 0x0, client_port = 12862, server_addr = 0x0, server_port = 80, transactions = 0x7f8215b6a140, messages = 0x7f81ee800db0, flags = 1 '\001', open_timestamp = {tv_sec = 1623407983, tv_usec = 32347}, close_timestamp = {tv_sec = 1623407983, tv_usec = 549597}, 
   in_data_counter = 39880, out_data_counter = 767} 
 (gdb) info locals 
 tx = <optimized out> 
 i = 41 
 n = 80 

 </code></pre> 

 Frame 2 
 <pre><code class="c"> 
 (gdb) fr 2 
 #2    0x00007f9175eb5ab2 in htp_connp_destroy_all (connp=0x7f81614a8b00) at htp_connection_parser.c:135 
 135 	 htp_connection_parser.c: No such file or directory. 
 (gdb) p *connp 
 $2 = {cfg = 0x7f917105b000, conn = 0x7f8219ba4810, user_data = 0x7f81ede90bc0, last_error = 0x0, in_status = HTP_STREAM_DATA, out_status = HTP_STREAM_DATA, out_data_other_at_tx_end = 0, in_timestamp = {tv_sec = 1623407983, tv_usec = 549597}, in_current_data = 0x0, 
   in_current_len = 0, in_current_read_offset = 0, in_current_consume_offset = 0, in_current_receiver_offset = 0, in_chunk_count = 17, in_chunk_request_index = 15, in_stream_offset = 39880, in_next_byte = -1, in_buf = 0x0, in_buf_size = 0, in_header = 0x0, in_tx = 0x0, 
   in_content_length = -1, in_body_data_left = -1, in_chunked_length = 0, in_state = 0x7f9175eba760 <htp_connp_REQ_IDLE>, in_state_previous = 0x7f9175eba760 <htp_connp_REQ_IDLE>, in_data_receiver_hook = 0x0, out_next_tx_index = 1, out_timestamp = {tv_sec = 1623407983, 
     tv_usec = 549597}, out_current_data = 0x0, out_current_len = 0, out_current_read_offset = 0, out_current_consume_offset = 0, out_current_receiver_offset = 0, out_stream_offset = 767, out_next_byte = -1, out_buf = 0x0, out_buf_size = 0, out_header = 0x0, out_tx = 0x0, 
   out_content_length = 530, out_body_data_left = 0, out_chunked_length = 0, out_state = 0x7f9175ebcc00 <htp_connp_RES_IDLE>, out_state_previous = 0x7f9175ebcc00 <htp_connp_RES_IDLE>, out_data_receiver_hook = 0x0, out_decompressor = 0x0, put_file = 0x0, req_decompressor = 0x0} 
 (gdb) info locals 
 No locals. 

 (gdb) fr 3 
 #3    0x000055cdf5485fea in HTPStateFree (state=0x7f81ede90bc0) at app-layer-htp.c:414 
 414 	 in app-layer-htp.c 
 (gdb) p *(HtpState *)state 
 $4 = {connp = 0x7f81614a8b00, conn = 0x7f8219ba4810, f = 0x7f8155932d80, transaction_cnt = 1, store_tx_id = 0, files_ts = 0x0, files_tc = 0x7f91053f12e0, cfg = 0x55cdf5b7f320 <cfglist>, flags = 6, events = 0, htp_messages_offset = 0, file_track_id = 1, 
   last_request_data_stamp = 39880, last_response_data_stamp = 767} 
 (gdb) info locals 
 tx_id = <optimized out> 
 total_txs = <optimized out> 
 s = 0x7f81ede90bc0 

 </code></pre> 


 Frame 4 
 <pre><code class="c"> 
 (gdb) fr 4 
 #4    0x000055cdf548ea00 in AppLayerParserStateProtoCleanup (pstate=0x7f81612ee740, alstate=<optimized out>, alproto=<optimized out>, protomap=<optimized out>) at app-layer-parser.c:1488 
 1488 	 app-layer-parser.c: No such file or directory. 
 (gdb) p *pstate 
 $5 = {flags = 96 '`', inspect_id = {80, 18}, log_id = 1, min_id = 1, decoder_events = 0x0} 

 </code></pre> 

 Frame 5 
 <pre><code class="c"> 
 (gdb) fr 5 
 #5    AppLayerParserStateCleanup (f=f@entry=0x7f8155932d80, alstate=<optimized out>, pstate=0x7f81612ee740) at app-layer-parser.c:1499 
 1499 	 in app-layer-parser.c 
 (gdb) p *f 
 $8 = {src = {address = {address_un_data32 = {3903619010, 0, 0, 0}, address_un_data16 = {32706, 59564, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\302\177\254\350", '\000' <repeats 11 times>}}, dst = {address = {address_un_data32 = {4029527692, 0, 0, 0}, address_un_data16 = {46732, 
         61485, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\214\266-\360", '\000' <repeats 11 times>}}, {sp = 12862, icmp_s = {type = 62 '>', code = 50 '2'}}, {dp = 80, icmp_d = {type = 80 'P', code = 0 '\000'}}, proto = 6 '\006', recursion_level = 0 '\000', vlan_id = {0, 0}, 
   use_cnt = 0, vlan_idx = 0 '\000', {{ffr_ts = 0 '\000', ffr_tc = 1 '\001'}, ffr = 16 '\020'}, timeout_at = 1623408043, thread_id = {32, 32}, next = 0x0, livedev = 0x0, flow_hash = 2135377145, lastts = {tv_sec = 1623407983, tv_usec = 549597}, timeout_policy = 60, flow_state = 2, 
   tenant_id = 0, probing_parser_toserver_alproto_masks = 0, probing_parser_toclient_alproto_masks = 0, flags = 1647387, file_flags = 4095, protodetect_dp = 0, parent_id = 0, m = {__data = {__lock = 1, __count = 0, __owner = 91690, __nusers = 1, __kind = 0, __spins = 0, 
       __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\001\000\000\000\000\000\000\000*f\001\000\001", '\000' <repeats 26 times>, __align = 1}, protoctx = 0x7f80cafe2980, protomap = 0 '\000', flow_end_flags = 16 '\020', alproto = 1, alproto_ts = 1, alproto_tc = 1, 
   alproto_orig = 0, alproto_expect = 0, de_ctx_version = 99, min_ttl_toserver = 54 '6', max_ttl_toserver = 54 '6', min_ttl_toclient = 62 '>', max_ttl_toclient = 255 '\377', alparser = 0x7f81612ee740, alstate = 0x7f81ede90bc0, sgh_toclient = 0x0, sgh_toserver = 0x0, flowvar = 0x0, 
   fb = 0x0, startts = {tv_sec = 1623407983, tv_usec = 32347}, todstpktcnt = 33, tosrcpktcnt = 19, todstbytecnt = 42198, tosrcbytecnt = 2257} 

 </code></pre> 

Back