Bug #1572
Updated by Victor Julien about 9 years ago
We're guessing this is the same issue as #1319, but with 2.0.8, and we have more information on it. We're seeing random crashes with suricate 2.0.8, approximately once or twice a day, in FlowGetKey for an icmp v4 destination unreachable packet. <pre> Program terminated with signal 11, Segmentation fault. #0 FlowGetKey (p=0xb8810750) at flow-hash.c:240 </pre> Representative backtrace is: <pre> (gdb) bt full #0 FlowGetKey (p=0xb8810750) at flow-hash.c:240 psrc = <value optimized out> pdst = <value optimized out> fhk = {{{src = 17297, dst = 768219096, sp = 36136, dp = 677, proto = 65010, recur = 23560, vlan_id = {6, 0}}, u32 = {17297, 768219096, 44404008, 1544093170, 6}}} hash = <value optimized out> key = <value optimized out> #1 FlowGetFlowFromHash (p=0xb8810750) at flow-hash.c:496 f = 0x0 key = <value optimized out> fb = <value optimized out> #2 0xb76171a6 in FlowHandlePacket (tv=0xb2f14580, p=0xb8810750) at flow.c:242 f = <value optimized out> #3 0xb756d05a in DecodeICMPV4 (tv=0xb2f14580, dtv=0xad902340, p=0xb8810750, pkt=0xb8810c32 "\003\004,4", len=60, pq=0xb0972328) at decode-icmpv4.c:195 icmp4eh = 0xb8810c32 #4 0xb756de57 in DecodeIPV4 (tv=0xb2f14580, dtv=0xad902340, p=0xb8810750, pkt=0xb8810c1e "E\033", len=80, pq=0xb0972328) at decode-ipv4.c:565 No locals. #5 0xb756c5f6 in DecodeEthernet (tv=0xb2f14580, dtv=0xad902340, p=0xb8810750, pkt=0xb8810c10 "", len=94, pq=0xb0972328) at decode-ethernet.c:60 No locals. #6 0xb766e348 in DecodePcap (tv=0xb2f14580, p=0xb8810750, data=0xad902340, pq=0xb0972328, postpq=0x0) at source-pcap.c:736 dtv = 0xad902340 __FUNCTION__ = "DecodePcap" #7 0xb7697547 in TmThreadsSlotVarRun (tv=0xb2f14580, p=0xb8810750, slot=0xb0972308) at tm-threads.c:559 SlotFunc = <value optimized out> r = <value optimized out> s = 0xb0972308 extra_p = <value optimized out> #8 0xb7670f82 in TmThreadsSlotProcessPkt (user=0xad900468 "\300\004\220\255\001", h=0xad07d13c, pkt=0xad6a7046 <Address 0xad6a7046 out of bounds>) at tm-threads.h:142 r = TM_ECODE_OK #9 PcapCallbackLoop (user=0xad900468 "\300\004\220\255\001", h=0xad07d13c, pkt=0xad6a7046 <Address 0xad6a7046 out of bounds>) at source-pcap.c:273 ptv = 0xad900468 p = 0xb8810750 current_time = {tv_sec = 1, tv_usec = -1391996728} #10 0xb73dafb3 in ?? () from /usr/lib/libpcap.so.1 No symbol table info available. #11 0xb73e2a24 in pcap_dispatch () from /usr/lib/libpcap.so.1 No symbol table info available. #12 0xb76703c0 in ReceivePcapLoop (tv=0xb2f14580, data=0xad900468, slot=0xb047a9c0) at source-pcap.c:318 packet_q_len = <value optimized out> ptv = 0xad900468 r = <value optimized out> s = 0xb047a9c0 __FUNCTION__ = "ReceivePcapLoop" #13 0xb7697088 in TmThreadsSlotPktAcqLoop (td=0xb2f14580) at tm-threads.c:703 tv = 0xb2f14580 (gdb) p *p $4 = {src = {family = 2 '\002', address = {address_un_data32 = {252738395, 0, 0, 0}, address_un_data16 = {31579, 3856, 0, 0, 0, 0, 0, 0}, address_un_data8 = "[{\020\017", '\000' <repeats 11 times>}}, dst = {family = 2 '\002', address = {address_un_data32 = {617224152, 0, 0, 0}, address_un_data16 = {6104, 9418, 0, 0, 0, 0, 0, 0}, address_un_data8 = "\330\027\312$", '\000' <repeats 11 times>}}, { sp = 3, type = 3 '\003'}, {dp = 4, code = 4 '\004'}, proto = 1 '\001', recursion_level = 0 '\000', vlan_id = {0, 0}, vlan_idx = 0 '\000', flowflags = 0 '\000', flags = 1048576, flow = 0x0, ts = { tv_sec = 1444144167, tv_usec = 72390}, {nfq_v = {id = 0, nfq_index = 0, verdicted = 0 '\000', mark = 0, ifi = 0, ifo = 0, hw_protocol = 0}, afp_v = {relptr = 0x0, copy_mode = 0, peer = 0x0, mpeer = 0x0}, pcap_v = {<No data fields>}}, ReleasePacket = 0xb76916d0 <PacketPoolReturnPacket>, pktvar = 0x0, ethh = 0xb8810c10, level3_comp_csum = -1, level4_comp_csum = -1, ip4h = 0xb8810c1e, ip6h = 0x0, {ip4vars = {comp_csum = 0, ip_src_u32 = 0, ip_dst_u32 = 0, ip_opts = {{type = 0 '\000', len = 0 '\000', data = 0x0} <repeats 40 times>}, ip_opt_cnt = 0 '\000', o_rr = 0x0, o_qs = 0x0, o_ts = 0x0, o_sec = 0x0, o_lsrr = 0x0, o_cipso = 0x0, o_sid = 0x0, o_ssrr = 0x0, o_rtralt = 0x0}, {ip6vars = {ip_opts_len = 0 '\000', l4proto = 0 '\000'}, ip6eh = {ip6fh = 0x0, fh_offset = 0, ip6rh = 0x0, ip6ah = 0x0, ip6eh = 0x0, ip6dh1 = 0x0, ip6dh2 = 0x0, ip6hh = 0x0, ip6hh_opt_hao = {ip6hao_type = 0 '\000', ip6hao_len = 0 '\000', ip6hao_hoa = {__in6_u = { __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}}, ip6hh_opt_ra = {ip6ra_type = 0 '\000', ip6ra_len = 0 '\000', ip6ra_value = 0}, ip6hh_opt_jumbo = {ip6j_type = 0 '\000', ip6j_len = 0 '\000', ip6j_payload_len = 0}, ip6dh1_opt_hao = {ip6hao_type = 0 '\000', ip6hao_len = 0 '\000', ip6hao_hoa = { __in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}}, ip6dh1_opt_ra = {ip6ra_type = 0 '\000', ip6ra_len = 0 '\000', ip6ra_value = 0}, ip6dh1_opt_jumbo = {ip6j_type = 0 '\000', ip6j_len = 0 '\000', ip6j_payload_len = 0}, ip6dh2_opt_hao = {ip6hao_type = 0 '\000', ip6hao_len = 0 '\000', ip6hao_hoa = { __in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}}, ip6dh2_opt_ra = {ip6ra_type = 0 '\000', ip6ra_len = 0 '\000', ip6ra_value = 0}, ip6dh2_opt_jumbo = {ip6j_type = 0 '\000', ip6j_len = 0 '\000', ip6j_payload_len = 0}, ip6_exthdrs = {{type = 0 '\000', next = 0 '\000', len = 0 '\000', data = 0x0} <repeats 40 times>}, ip6_exthdrs_cnt = 0 '\000'}}}, {tcpvars = {tcp_opt_cnt = 0 '\000', tcp_opts = {{type = 8 '\b', len = 10 '\n', data = 0xb8810c4a "sy&\n\\[\234\263\021%$\237/\212\200"}, {type = 3 '\003', len = 3 '\003', data = 0xb8810c4d "\n\\[\234\263\021%$\237/\212\200"}, {type = 4 '\004', len = 2 '\002', data = 0x0}, {type = 3 '\003', len = 3 '\003', data = 0xb8810c59 ""}, {type = 76 'L', len = 4 '\004', data = 0xb8810c5e "\305\177 J\214@\302C$"}, {type = 0 '\000', len = 0 '\000', data = 0x0} <repeats 15 times>}, ts = 0x0, sack = 0x0, sackok = 0x0, ws = 0x0, mss = 0x0}, udpvars = {<No data fields>}, icmpv4vars = {id = 0, seq = 0, mtu = 2568, error_ptr = 3095465034, emb_ipv4h = 0x303, emb_tcph = 0xb8810c4d, emb_udph = 0x204, emb_icmpv4h = 0x0, emb_ip4_src = {s_addr = 771}, emb_ip4_dst = {s_addr = 3095465049}, emb_ip4_hlen = 76 'L', emb_ip4_proto = 4 '\004', emb_sport = 0, emb_dport = 3166}, icmpv6vars = {id = 0, seq = 0, mtu = 2568, error_ptr = 3095465034, emb_ipv6h = 0x303, emb_tcph = 0xb8810c4d, emb_udph = 0x204, emb_icmpv6h = 0x0, emb_ip6_src = { 771, 3095465049, 1100, 3095465054}, emb_ip6_dst = {0, 0, 0, 0}, emb_ip6_proto_next = 0 '\000', emb_sport = 0, emb_dport = 0}}, tcph = 0x0, udph = 0x0, sctph = 0x0, icmpv4h = 0xb8810c32, icmpv6h = 0x0, ppph = 0x0, pppoesh = 0x0, pppoedh = 0x0, greh = 0x0, vlanh = {0x0, 0x0}, payload = 0xb8810c3a "\245\373\260S\033\327\\\241\260\237LW\330\027\312$sy&\n\\[\234\263\021%$\237/\212\200", payload_len = 52, action = 0 '\000', pkt_src = 1 '\001', pktlen = 94, ext_pkt = 0x0, livedev = 0xb7de6d58, alerts = {cnt = 0, alerts = {{num = 0, order_id = 0, action = 0 '\000', flags = 0 '\000', s = 0x0, tx_id = 0} <repeats 15 times>}}, host_src = 0x0, host_dst = 0x0, pcap_cnt = 0, events = {cnt = 1 '\001', events = "\022g", '\000' <repeats 12 times>}, app_layer_events = 0x0, next = 0x0, prev = 0x0, datalink = 1, debuglog_flowbits_names_len = 0, debuglog_flowbits_names = 0x0, root = 0x0, tunnel_mutex = {__data = {__lock = 0, __count = 0, __owner = 0, __kind = 0, __nusers = 0, {__spins = 0, __list = {__next = 0x0}}}, __size = '\000' <repeats 23 times>, __align = 0}, tunnel_rtv_cnt = 0, tunnel_tpr_cnt = 0} </pre> Seems that DecodeICMPV4 is calling DecodePartialIPV4, which sets p->icmpv4vars.emb_ipv4h to the headers correctly. But at some time afterwards (and before FlowGetKey) the pointer of emb_ipv4h gets corrupted (as does the values after it in the structure).