Description
We have seen the following stack in production. The problem is the migration fails, but we set the netvc to null before calling do_io_close. This causes the get_server_ip() call in HttpServerSession::do_io_close to dereference a NULL.
(gdb) bt #0 0x00002aae5acc2625 in raise () from /lib64/libc.so.6 #1 0x00002aae5acc3d8d in abort () from /lib64/libc.so.6 #2 0x00002aae58062149 in ink_die_die_die () at ../../../../trafficserver/lib/ts/ink_error.cc:43 #3 0x00002aae58062202 in ink_fatal_va(const char *, typedef __va_list_tag __va_list_tag *) (fmt=0x2aae58077b18 "%s:%d: failed assert `%s`", ap=0x2aae62dccd68) at ../../../../trafficserver/lib/ts/ink_error.cc:65 #4 0x00002aae580622a1 in ink_fatal (message_format=0x2aae58077b18 "%s:%d: failed assert `%s`") at ../../../../trafficserver/lib/ts/ink_error.cc:73 #5 0x00002aae5805fa06 in _ink_assert (expression=0x7c2a51 "server_vc != NULL", file=0x7c2a10 "../../../../trafficserver/proxy/http/../http/HttpServerSession.h", line=123) at ../../../../trafficserver/lib/ts/ink_assert.cc:37 #6 0x00000000006038b9 in HttpServerSession::get_server_ip (this=0x2aac08b0efd0) at ../../../../trafficserver/proxy/http/../http/HttpServerSession.h:123 #7 0x000000000060801e in HttpServerSession::do_io_close (this=0x2aac08b0efd0, alerrno=-1) at ../../../../trafficserver/proxy/http/HttpServerSession.cc:130 #8 0x0000000000609b12 in HttpSessionManager::acquire_session (this=0xae1ba0, ip=0x2aac2f2817c8, hostname=0x2aac305e8a19 "sc1.ycpi.vip.bf1.yahoo.com", ua_session=0x2aac104fca70, sm=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSessionManager.cc:311 #9 0x00000000005f7554 in HttpSM::do_http_server_open (this=0x2aac2f2810b0, raw=false) at ../../../../trafficserver/proxy/http/HttpSM.cc:4872 #10 0x0000000000600357 in HttpSM::set_next_state (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7385 #11 0x00000000005ff4ee in HttpSM::call_transact_and_set_next_state (this=0x2aac2f2810b0, f=0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7198 #12 0x00000000005ef0d5 in HttpSM::state_cache_open_write (this=0x2aac2f2810b0, event=1108, data=0x2aaad0203540) at ../../../../trafficserver/proxy/http/HttpSM.cc:2581 #13 0x00000000005ef79c in HttpSM::main_handler (this=0x2aac2f2810b0, event=1108, data=0x2aaad0203540) at ../../../../trafficserver/proxy/http/HttpSM.cc:2693 #14 0x000000000051381a in Continuation::handleEvent (this=0x2aac2f2810b0, event=1108, data=0x2aaad0203540) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #15 0x00000000005d9e38 in HttpCacheSM::state_cache_open_write (this=0x2aac2f282ba0, event=1108, data=0x2aaad0203540) at ../../../../trafficserver/proxy/http/HttpCacheSM.cc:167 #16 0x000000000051381a in Continuation::handleEvent (this=0x2aac2f282ba0, event=1108, data=0x2aaad0203540) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #17 0x000000000073c9be in CacheVC::callcont (this=0x2aaad0203540, event=1108) at ../../../../trafficserver/iocore/cache/P_CacheInternal.h:673 #18 0x0000000000746aca in Cache::open_write (this=0x2aaadc008c90, cont=0x2aac2f282ba0, key=0x2aae62dcd650, info=0x0, apin_in_cache=0, type=CACHE_FRAG_TYPE_HTTP, hostname=0x2aac0cad3405 "68.media.tumblr.com9548920493ae47f3954b2a04b9d8763a/tumblr_inline_n1gft6Y4Vj1qjk6k8.jpg", host_len=19) at ../../../../trafficserver/iocore/cache/CacheWrite.cc:1789 #19 0x0000000000723667 in Cache::open_write (this=0x2aaadc008c90, cont=0x2aac2f282ba0, url=0x2aac2f281158, request=0x2aac2f281828, old_info=0x0, pin_in_cache=0, type=CACHE_FRAG_TYPE_HTTP) at ../../../../trafficserver/iocore/cache/P_CacheInternal.h:1104 #20 0x00000000007211be in CacheProcessor::open_write (this=0x1059780, cont=0x2aac2f282ba0, expected_size=0, url=0x2aac2f281158, cluster_cache_local=false, request=0x2aac2f281828, old_info=0x0, pin_in_cache=0, type=CACHE_FRAG_TYPE_HTTP) at ../../../../trafficserver/iocore/cache/Cache.cc:3701 #21 0x00000000005da25e in HttpCacheSM::open_write (this=0x2aac2f282ba0, url=0x2aac2f281158, request=0x2aac2f281828, old_info=0x0, pin_in_cache=0, retry=true, allow_multiple=false) at ../../../../trafficserver/proxy/http/HttpCacheSM.cc:298 #22 0x00000000005f672b in HttpSM::do_cache_prepare_action (this=0x2aac2f2810b0, c_sm=0x2aac2f282ba0, object_read_info=0x0, retry=true, allow_multiple=false) at ../../../../trafficserver/proxy/http/HttpSM.cc:4686 #23 0x000000000060648d in HttpSM::do_cache_prepare_write (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:4611 #24 0x0000000000600702 in HttpSM::set_next_state (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7462 #25 0x00000000005ff4ee in HttpSM::call_transact_and_set_next_state (this=0x2aac2f2810b0, f=0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7198 #26 0x00000000005ebfc2 in HttpSM::handle_api_return (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1649 #27 0x00000000005ebe35 in HttpSM::state_api_callout (this=0x2aac2f2810b0, event=60000, data=0x0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1587 #28 0x00000000005eb531 in HttpSM::state_api_callback (this=0x2aac2f2810b0, event=60000, data=0x0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1383 #29 0x00000000005335d3 in TSHttpTxnReenable (txnp=0x2aac2f2810b0, event=60000) at ../../../trafficserver/proxy/InkAPI.cc:5647 #30 0x00002aaacc66d4f9 in on_OS_DNS (req=0x2aaad8568570, txnp=@0x2aae62dcdbb0) at trafficserver/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc:139 #31 0x00002aaacc66d9cd in collapsed_cont (contp=0x2aac3c74db10, event=TS_EVENT_HTTP_OS_DNS, edata=0x2aac2f2810b0) at trafficserver/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc:240 #32 0x0000000000527f1e in INKContInternal::handle_event (this=0x2aac3c74db10, event=60003, edata=0x2aac2f2810b0) at ../../../trafficserver/proxy/InkAPI.cc:1017 #33 0x000000000051381a in Continuation::handleEvent (this=0x2aac3c74db10, event=60003, data=0x2aac2f2810b0) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #34 0x00000000005286c1 in APIHook::invoke (this=0x2aaadc6d5070, event=60003, edata=0x2aac2f2810b0) at ../../../trafficserver/proxy/InkAPI.cc:1228 #35 0x00000000005ebb74 in HttpSM::state_api_callout (this=0x2aac2f2810b0, event=0, data=0x0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1505 #36 0x00000000005f8719 in HttpSM::do_api_callout_internal (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:5154 #37 0x00000000006063d9 in HttpSM::do_api_callout (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:460 #38 0x00000000005ff55c in HttpSM::set_next_state (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7231 #39 0x00000000005ff4ee in HttpSM::call_transact_and_set_next_state (this=0x2aac2f2810b0, f=0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7198 #40 0x00000000005f4848 in HttpSM::do_hostdb_lookup (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:4183 #41 0x0000000000600053 in HttpSM::set_next_state (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7329 #42 0x00000000005ff4ee in HttpSM::call_transact_and_set_next_state (this=0x2aac2f2810b0, f=0) at ../../../../trafficserver/proxy/http/HttpSM.cc:7198 #43 0x00000000005ebfc2 in HttpSM::handle_api_return (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1649 #44 0x00000000005ebe35 in HttpSM::state_api_callout (this=0x2aac2f2810b0, event=60000, data=0x0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1587 #45 0x00000000005eb531 in HttpSM::state_api_callback (this=0x2aac2f2810b0, event=60000, data=0x0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1383 #46 0x00000000005335d3 in TSHttpTxnReenable (txnp=0x2aac2f2810b0, event=60000) at ../../../trafficserver/proxy/InkAPI.cc:5647 #47 0x00002aaaaab2d2d8 in main_handler (cont=<value optimized out>, event=<value optimized out>, edata=0x2aac2f2810b0) at trafficserver/plugins/experimental/regex_revalidate/regex_revalidate.c:479 #48 0x0000000000527f1e in INKContInternal::handle_event (this=0x1ab8a80, event=60015, edata=0x2aac2f2810b0) at ../../../trafficserver/proxy/InkAPI.cc:1017 #49 0x000000000051381a in Continuation::handleEvent (this=0x1ab8a80, event=60015, data=0x2aac2f2810b0) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #50 0x00000000005286c1 in APIHook::invoke (this=0x1ab9cc0, event=60015, edata=0x2aac2f2810b0) at ../../../trafficserver/proxy/InkAPI.cc:1228 #51 0x00000000005ebb74 in HttpSM::state_api_callout (this=0x2aac2f2810b0, event=0, data=0x0) at ../../../../trafficserver/proxy/http/HttpSM.cc:1505 #52 0x00000000005f8719 in HttpSM::do_api_callout_internal (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:5154 #53 0x00000000006063d9 in HttpSM::do_api_callout (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:460 #54 0x000000000060641c in HttpSM::setup_cache_lookup_complete_api (this=0x2aac2f2810b0) at ../../../../trafficserver/proxy/http/HttpSM.cc:2590 #55 0x00000000005ef545 in HttpSM::state_cache_open_read (this=0x2aac2f2810b0, event=1103, data=0xffffffffffffb050) at ../../../../trafficserver/proxy/http/HttpSM.cc:2651 #56 0x00000000005ef79c in HttpSM::main_handler (this=0x2aac2f2810b0, event=1103, data=0xffffffffffffb050) at ../../../../trafficserver/proxy/http/HttpSM.cc:2693 #57 0x000000000051381a in Continuation::handleEvent (this=0x2aac2f2810b0, event=1103, data=0xffffffffffffb050) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #58 0x00000000005d9c77 in HttpCacheSM::state_cache_open_read (this=0x2aac2f282ba0, event=1103, data=0xffffffffffffb050) at ../../../../trafficserver/proxy/http/HttpCacheSM.cc:131 #59 0x000000000051381a in Continuation::handleEvent (this=0x2aac2f282ba0, event=1103, data=0xffffffffffffb050) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #60 0x000000000073c0d9 in CacheVC::openReadStartHead (this=0x2aabec612e50, event=2, e=0x2aac7c2c1550) at ../../../../trafficserver/iocore/cache/CacheRead.cc:1209 #61 0x00000000007374a4 in CacheVC::openReadFromWriter (this=0x2aabec612e50, event=2, e=0x2aac7c2c1550) at ../../../../trafficserver/iocore/cache/CacheRead.cc:324 #62 0x000000000051381a in Continuation::handleEvent (this=0x2aabec612e50, event=2, data=0x2aac7c2c1550) at ../../../trafficserver/iocore/eventsystem/I_Continuation.h:150 #63 0x0000000000794d1d in EThread::process_event (this=0x2aae60868010, e=0x2aac7c2c1550, calling_code=2) at ../../../../trafficserver/iocore/eventsystem/UnixEThread.cc:145 #64 0x0000000000795223 in EThread::execute_regular (this=0x2aae60868010) at ../../../../trafficserver/iocore/eventsystem/UnixEThread.cc:242 #65 0x0000000000795466 in EThread::execute (this=0x2aae60868010) at ../../../../trafficserver/iocore/eventsystem/UnixEThread.cc:304 #66 0x00000000007940bf in spawn_thread_internal (a=0x13102c0) at ../../../../trafficserver/iocore/eventsystem/Thread.cc:85 #67 0x00002aae590d8aa1 in start_thread () from /lib64/libpthread.so.0 #68 0x00002aae5ad7893d in clone () from /lib64/libc.so.6
Attachments
Issue Links
- links to