00010000:00000001:0.0F:1789466098.663775:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466098.663804:0:222999:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789466098.663806:0:222999:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789466098.663809:0:222999:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466098.663815:0:222999:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789466098.663816:0:222999:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466098.663818:0:222999:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466098.663821:0:222999:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0F:1789466098.664737:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789466098.664746:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789466098.664750:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:3.0F:1789466098.821862:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789466098.821886:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789466098.821890:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789466098.821893:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789466098.821904:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:3.0:1789466098.821922:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94700 00000400:00000200:3.0:1789466098.821930:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 4704 00000800:00000001:3.0:1789466098.821937:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789466098.821940:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789466098.821944:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466098.821949:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:3.0:1789466098.821956:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:3.0:1789466098.821958:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:3.0:1789466098.821964:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8966d3fe1880. 00000100:00000040:3.0:1789466098.821967:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff8966d3fe1880 x1876386402354944 msgsize 224 00000100:00100000:3.0:1789466098.821972:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:3.0:1789466098.822708:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:3.0:1789466098.823006:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789466098.823016:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789466098.823019:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789466098.823022:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789466098.823029:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:3.0:1789466098.823067:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94680 00000400:00000200:3.0:1789466098.823073:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 4928 00000800:00000001:3.0:1789466098.823079:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789466098.823087:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789466098.823089:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466098.823093:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:3.0:1789466098.823098:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:3.0:1789466098.823099:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:3.0:1789466098.823104:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8966d3fe0a80. 00000100:00000040:3.0:1789466098.823107:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff8966d3fe0a80 x1876386402354816 msgsize 224 00000100:00100000:3.0:1789466098.823112:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0F:1789466098.825156:0:121175:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1789466098.828077:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:3.0:1789466098.828111:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1789466098.828113:0:201492:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:3.0:1789466098.828119:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1789466098.829646:0:121175:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402354944 02000000:00000001:2.0:1789466098.829653:0:121175:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1789466098.829655:0:121175:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789466098.829658:0:121175:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789466098.829663:0:121175:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789466098.829667:0:121175:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402354944 00000020:00000001:2.0:1789466098.829671:0:121175:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1789466098.829672:0:121175:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b8705ac 00000020:00000001:2.0:1789466098.829675:0:121175:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1789466098.829679:0:121175:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5cc5000 refcount=7 00000020:00000001:2.0:1789466098.829683:0:121175:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000020:00000001:2.0:1789466098.829685:0:121175:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000100:00000001:2.0:1789466098.829690:0:121175:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1789466098.829696:0:121175:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ede8e440. 00000020:00000010:2.0:1789466098.829701:0:121175:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967f014f940. 00000020:00000040:2.0:1789466098.829708:0:121175:0:(obd_config.c:995:class_incref()) incref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:2.0:1789466098.829724:0:121175:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:2.0:1789466098.829731:0:121175:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967f190a800. 00000020:00000010:2.0:1789466098.829736:0:121175:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967e983d580. 00000020:00000010:2.0:1789466098.829741:0:121175:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8966ccee5e10. 00000100:00000040:2.0:1789466098.829752:0:121175:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00000001:2.0:1789466098.829756:0:121175:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1789466098.829757:0:121175:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1789466098.829759:0:121175:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789466098.829766:0:121175:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1789466098.829769:0:121175:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1789466098.829770:0:121175:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789466098.829781:0:121175:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1789466098.829828:0:201492:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402354816 02000000:00000001:0.0:1789466098.829831:0:201492:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1789466098.829833:0:201492:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789466098.829834:0:201492:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466098.829837:0:201492:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466098.829839:0:201492:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402354816 00000020:00000001:0.0:1789466098.829841:0:201492:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466098.829842:0:201492:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b86f3e6 00000020:00000001:0.0:1789466098.829843:0:201492:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466098.829846:0:201492:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967c3b5d800 refcount=7 00000020:00000001:0.0:1789466098.829848:0:201492:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000020:00000001:0.0:1789466098.829850:0:201492:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000100:00000001:0.0:1789466098.829852:0:201492:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466098.829856:0:201492:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e880. 00000020:00000010:0.0:1789466098.829859:0:201492:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967f014fb80. 00000020:00000040:0.0:1789466098.829867:0:201492:0:(obd_config.c:995:class_incref()) incref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000001:0.0:1789466098.829869:0:201492:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:0.0:1789466098.829873:0:201492:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff89670c1fec00. 00000020:00000010:0.0:1789466098.829877:0:201492:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3528f00. 00000020:00000010:0.0:1789466098.829879:0:201492:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967e94d8d48. 00000100:00000001:0.0:1789466098.829883:0:201492:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1789466098.829884:0:201492:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1789466098.829886:0:201492:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466098.829889:0:201492:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1789466098.829891:0:201492:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1789466098.829892:0:201492:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1789466098.829893:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000020:00000040:3.0:1789466098.829897:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000001:0.0:1789466098.829899:0:201492:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1789466098.829900:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000020:00000040:3.0:1789466098.829902:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:3.0:1789466098.829925:0:201490:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1789466098.829928:0:201490:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1789466098.829935:0:201490:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3501 00000100:00000040:3.0:1789466098.829939:0:201490:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967f5cc5000 : new rpc_count 1 00000100:00000001:3.0:1789466098.829941:0:201490:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613673469155456 : -130400240396160 : ffff8966d3fe1880) 00000100:00000040:3.0:1789466098.829948:0:201490:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff8966d3fe1880 x1876386402354944/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:594/0 lens 224/0 e 0 to 0 dl 1789466109 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466098.829957:0:201490:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466098.829961:0:201490:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657c740. 00000100:00000001:3.0:1789466098.829964:0:201490:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1789466098.829966:0:201490:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967c657c740. 00000100:00100000:3.0:1789466098.829971:0:201490:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff8966d3fe1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402354944:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:3.0:1789466098.829975:0:201490:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402354944 00000020:00000001:3.0:1789466098.829978:0:201490:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466098.829982:0:201490:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1789466098.829982:0:201492:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000020:00000001:3.0:1789466098.829985:0:201490:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466098.829987:0:201490:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1789466098.829989:0:201490:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000100:00000001:0.0:1789466098.829989:0:201492:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:3.0:1789466098.829992:0:201490:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00100000:0.0:1789466098.829992:0:201492:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3502 00000020:00000001:3.0:1789466098.829995:0:201490:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1789466098.829995:0:201492:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967c3b5d800 : new rpc_count 1 00000100:00000001:0.0:1789466098.829997:0:201492:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613673469151872 : -130400240399744 : ffff8966d3fe0a80) 00000020:00000001:3.0:1789466098.829998:0:201490:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1789466098.830000:0:201490:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:0.0:1789466098.830001:0:201492:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff8966d3fe0a80 x1876386402354816/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:594/0 lens 224/0 e 0 to 0 dl 1789466109 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1789466098.830003:0:201490:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466098.830005:0:201490:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1789466098.830009:0:201490:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1789466098.830009:0:201492:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466098.830011:0:201492:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e200. 02000000:00000001:3.0:1789466098.830012:0:201490:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1789466098.830014:0:201492:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1789466098.830016:0:201492:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967ee05e200. 00000100:00100000:0.0:1789466098.830019:0:201492:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff8966d3fe0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_016:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402354816:12345-192.168.203.42@tcp:400:kworker.0 02000000:00000010:3.0:1789466098.830020:0:201490:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c7459c00. 02000000:00000001:3.0:1789466098.830022:0:201490:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1789466098.830023:0:201492:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402354816 00000100:00000001:3.0:1789466098.830025:0:201490:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466098.830025:0:201492:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466098.830027:0:201490:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466098.830027:0:201492:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1789466098.830028:0:201492:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1789466098.830030:0:201490:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 12884913986, transno 0, xid 1876386402354944 00000020:00000001:0.0:1789466098.830030:0:201492:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1789466098.830031:0:201492:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00010000:00000001:3.0:1789466098.830033:0:201490:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00000020:00000001:0.0:1789466098.830033:0:201492:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1789466098.830035:0:201492:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1789466098.830050:0:201492:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00010000:00000200:3.0:1789466098.830051:0:201490:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff8966d3fe1880 x1876386402354944/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:594/0 lens 224/224 e 0 to 0 dl 1789466109 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1789466098.830051:0:201492:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1789466098.830053:0:201492:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466098.830055:0:201492:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1789466098.830057:0:201492:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1789466098.830058:0:201492:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 00010000:00000001:3.0:1789466098.830060:0:201490:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 02000000:00000010:0.0:1789466098.830061:0:201492:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967ee8fe400. 00010000:00000001:3.0:1789466098.830062:0:201490:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466098.830063:0:201492:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466098.830064:0:201492:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789466098.830065:0:201490:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=7 v=5 (1 1 1 1) 00000020:00000001:0.0:1789466098.830066:0:201492:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1789466098.830068:0:201492:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 17179870928, transno 0, xid 1876386402354816 00000100:00000001:3.0:1789466098.830070:0:201490:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00010000:00000001:0.0:1789466098.830070:0:201492:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00000100:00000040:3.0:1789466098.830072:0:201490:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00010000:00000200:0.0:1789466098.830073:0:201492:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff8966d3fe0a80 x1876386402354816/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:594/0 lens 224/224 e 0 to 0 dl 1789466109 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466098.830075:0:201490:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:3.0:1789466098.830077:0:201490:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1789466098.830079:0:201490:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466098.830082:0:201490:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789466098.830086:0:201490:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00010000:00000001:0.0:1789466098.830089:0:201492:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00000400:00000010:3.0:1789466098.830090:0:201490:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed93ab28. 00010000:00000001:0.0:1789466098.830091:0:201492:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789466098.830093:0:201492:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=7 v=5 (1 1 1 1) 00000100:00000001:0.0:1789466098.830095:0:201492:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1789466098.830096:0:201490:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402354944, offset 224 00000100:00000040:0.0:1789466098.830097:0:201492:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 6 to 192.168.203.42@tcp 00000100:00000001:0.0:1789466098.830099:0:201492:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:0.0:1789466098.830101:0:201492:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:3.0:1789466098.830102:0:201490:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 02000000:00000001:0.0:1789466098.830102:0:201492:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466098.830104:0:201492:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789466098.830106:0:201492:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:0.0:1789466098.830108:0:201492:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967f7b93440. 00000100:00000200:0.0:1789466098.830110:0:201492:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402354816, offset 224 00000400:00000200:0.0:1789466098.830113:0:201492:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:3.0:1789466098.830115:0:201490:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:3.0:1789466098.830136:0:201490:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93553:93553:256:4294967295] 192.168.203.42@tcp LPNI seq info [93549:93549:8:4294967295] 00000400:00000200:3.0:1789466098.830151:0:201490:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:3.0:1789466098.830157:0:201490:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000400:00000200:0.0:1789466098.830157:0:201492:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:0.0:1789466098.830161:0:201492:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93554:93554:255:4294967295] 192.168.203.42@tcp LPNI seq info [93550:93550:7:4294967295] 00000400:00000200:0.0:1789466098.830168:0:201492:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000010:3.0:1789466098.830171:0:201490:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966cb0f4100. 00000800:00000200:0.0:1789466098.830172:0:201492:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:0.0:1789466098.830175:0:201492:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8967c9f9ec00. 00000800:00000200:3.0:1789466098.830178:0:201490:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:0.0:1789466098.830178:0:201492:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:3.0:1789466098.830186:0:201490:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:0.0:1789466098.830186:0:201492:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:3.0:1789466098.830191:0:201490:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966cb0f4100 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:0.0:1789466098.830191:0:201492:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8967c9f9ec00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789466098.830201:0:201490:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466098.830201:0:201492:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789466098.830204:0:201490:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00000100:00000040:0.0:1789466098.830204:0:201492:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:0.0:1789466098.830206:0:201492:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:0.0:1789466098.830208:0:201492:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466098.830210:0:201492:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00010000:00000001:3.0:1789466098.830211:0:201490:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:3.0:1789466098.830212:0:201490:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466098.830213:0:201490:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1789466098.830214:0:201492:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8966d3fe0a80 x1876386402354816/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:594/0 lens 224/224 e 0 to 0 dl 1789466109 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789466098.830217:0:201490:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8966d3fe1880 x1876386402354944/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:594/0 lens 224/224 e 0 to 0 dl 1789466109 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789466098.830224:0:201492:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff8966d3fe0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_016:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402354816:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 213us (7114us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466098.830225:0:201490:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff8966d3fe1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402354944:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 267us (8256us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466098.830233:0:201490:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3501 00000100:00000040:3.0:1789466098.830236:0:201490:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967f5cc5000 : new rpc_count 0 00000100:00100000:0.0:1789466098.830236:0:201492:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3502 00000100:00000001:3.0:1789466098.830238:0:201490:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000040:0.0:1789466098.830239:0:201492:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967c3b5d800 : new rpc_count 0 00000100:00000001:3.0:1789466098.830240:0:201490:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1789466098.830240:0:201492:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1789466098.830241:0:201492:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1789466098.830244:0:201490:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967e983d580. 00000020:00000010:0.0:1789466098.830244:0:201492:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3528f00. 00000020:00000010:3.0:1789466098.830248:0:201490:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8966ccee5e10. 00000020:00000010:0.0:1789466098.830248:0:201492:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967e94d8d48. 00000020:00000010:3.0:1789466098.830252:0:201490:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967f190a800. 00000020:00000010:0.0:1789466098.830252:0:201492:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff89670c1fec00. 00000020:00000040:3.0:1789466098.830258:0:201490:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5cc5000 : new refcount 6 00000020:00000040:0.0:1789466098.830258:0:201492:0:(genops.c:879:class_export_put()) PUTting export ffff8967c3b5d800 : new refcount 6 00000100:00000001:3.0:1789466098.830260:0:201490:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466098.830260:0:201492:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1789466098.830300:0:117757:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789466098.830304:0:117757:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966cb0f4100. 00000400:00000200:3.0:1789466098.830309:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466098.830313:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:3.0:1789466098.830319:0:117757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed93ab28 00000400:00000010:3.0:1789466098.830321:0:117757:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed93ab28. 00000100:00000001:3.0:1789466098.830325:0:117757:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:3.0:1789466098.830326:0:117757:0:(events.c:402:reply_out_callback()) Process leaving 00000800:00000200:3.0:1789466098.830685:0:117757:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789466098.830688:0:117757:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8967c9f9ec00. 00000400:00000200:3.0:1789466098.830692:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466098.830696:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:3.0:1789466098.830700:0:117757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967f7b93440 00000400:00000010:3.0:1789466098.830702:0:117757:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967f7b93440. 00000100:00000001:3.0:1789466098.830704:0:117757:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:3.0:1789466098.830706:0:117757:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00000040:0.0:1789466098.855115:0:222999:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37000 (cpu 0) has reached its deadline 4313013881, at 4313014272 00000100:00000040:2.0:1789466098.855117:0:201793:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37220 (cpu 2) has reached its deadline 4313013881, at 4313014272 00000100:00000010:2.0:1789466098.855132:0:201793:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8966d491a300. 00000100:00000010:0.0:1789466098.855135:0:222999:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8966d9c24de0. 00000100:00000040:3.0:1789466098.855230:0:207024:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37330 (cpu 3) has reached its deadline 4313013881, at 4313014272 00000100:00000010:3.0:1789466098.855247:0:207024:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8966d9c24ae0. 00000100:00000040:1.0:1789466098.856686:0:222274:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37110 (cpu 1) has reached its deadline 4313013881, at 4313014273 00000100:00000010:1.0:1789466098.856699:0:222274:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8966d9c249c0. 00000100:00000001:1.0:1789466099.431599:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466099.431614:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466099.431615:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466099.431620:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466099.431622:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466099.431626:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466099.431629:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466099.688810:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466099.688817:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466099.688839:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466099.688842:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466099.688844:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466100.455115:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466100.455130:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466100.455132:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466100.455137:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466100.455139:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466100.455144:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466100.455146:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466100.711180:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466100.711187:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466100.712467:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466100.712472:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466100.712477:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00080000:00000010:3.0:1789466101.334808:0:224681:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3614e80. 00080000:00000010:3.0:1789466101.334818:0:224681:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3614e80. 00000020:00000001:3.0:1789466101.334824:0:224681:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:3.0:1789466101.334832:0:224681:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3614e80. 00080000:00000010:3.0:1789466101.334836:0:224681:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3614e80. 00000020:00000001:3.0:1789466101.334840:0:224681:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466101.479588:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466101.479605:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466101.479607:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466101.479611:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466101.479614:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466101.479619:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466101.479622:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466101.735174:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466101.735181:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466101.736156:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789466101.736161:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789466101.736165:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00080000:00000010:0.0:1789466102.162936:0:224716:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3528080. 00080000:00000010:0.0:1789466102.162947:0:224716:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3528080. 00000020:00000001:0.0:1789466102.162953:0:224716:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:0.0:1789466102.162961:0:224716:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3528080. 00080000:00000010:0.0:1789466102.162966:0:224716:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3528080. 00000020:00000001:0.0:1789466102.163210:0:224716:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466102.503085:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466102.503096:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466102.503098:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466102.503101:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466102.503104:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466102.503108:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466102.503111:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466102.765957:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466102.765964:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466102.771025:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789466102.771031:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789466102.771826:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00080000:00000010:1.0:1789466103.044415:0:224759:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d832ae80. 00080000:00000010:1.0:1789466103.044426:0:224759:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d832ae80. 00000020:00000001:1.0:1789466103.044432:0:224759:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:1.0:1789466103.044442:0:224759:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d832ae80. 00080000:00000010:1.0:1789466103.044453:0:224759:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d832ae80. 00000020:00000001:1.0:1789466103.044458:0:224759:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1789466103.464724:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789466103.464742:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789466103.464746:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789466103.464749:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789466103.464752:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789466103.464755:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789466103.464758:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789466103.464761:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789466103.464764:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789466103.464766:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1789466103.464772:0:134719:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18356) 00000100:00000001:1.0:1789466103.529889:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466103.529897:0:117762:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1789466103.529910:0:117762:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789466093/real 1789466093] req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1789466103.529928:0:117762:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789466103.529930:0:117762:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789466103.529933:0:117762:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8967cbd79420. 00000400:00000200:1.0:1789466103.529940:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967f7b93e58 00000400:00000010:1.0:1789466103.529943:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967f7b93e58. 00000100:00000001:1.0:1789466103.529947:0:117762:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789466103.529950:0:117762:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789466103.529959:0:117762:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.529966:0:117762:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789466103.529968:0:117762:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.529970:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466103.529972:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1789466103.529976:0:117762:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.529983:0:117762:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.529992:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466103.529994:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.529997:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466103.530000:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466103.530002:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789466103.530006:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530013:0:117762:0:(client.c:2143:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1789466103.530016:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.530026:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466103.530027:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.530030:0:117762:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1789466103.530032:0:117762:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1789466103.530050:0:117762:0:(import.c:65:import_set_state_nolock()) ffff8967e7ba6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789466103.530073:0:117762:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00000001:1.0:1789466103.530075:0:117762:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1789466103.530080:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466103.530089:0:117762:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8967cb941880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:4861e2df-2452-4fd5-a0b1-355ea409d38c:117762:1876386404768640:0@lo:250:kworker.0 00000100:00000001:1.0:1789466103.530095:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530098:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466103.530101:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8967cb941880 x1876386404768640/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466103 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530108:0:117762:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789466103.530110:0:117762:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789466103.530114:0:117762:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8967ee8fe000. 02000000:00000001:1.0:1789466103.530121:0:117762:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789466103.530124:0:117762:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789466103.530126:0:117762:0:(genops.c:1222:class_import_put()) import ffff8967e7ba6800 refcount=3 obd=MGC192.168.203.142@tcp 00000020:00000001:1.0:1789466103.530128:0:117762:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789466103.530131:0:117762:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8967ee8fd000. 02000000:00000001:1.0:1789466103.530135:0:117762:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789466103.530137:0:117762:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789466103.530139:0:117762:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8967cb941880. 00000100:00000001:1.0:1789466103.530143:0:117762:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789466103.530144:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.530146:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1789466103.530164:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466103.530169:0:223621:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1789466103.530172:0:223621:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1789466103.530174:0:223621:0:(import.c:65:import_set_state_nolock()) ffff8967e7ba6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1789466103.530183:0:223621:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1789466103.530187:0:223621:0:(import.c:545:import_select_connection()) MGC192.168.203.142@tcp: connect to NID 0@lo last attempt 18341 00000100:00001000:1.0:1789466103.530191:0:223621:0:(import.c:2196:obd_at_measure()) add 10 to ffff8967e7ba6c18 time=142 v=5 (1 1 0 0) 00000100:00001000:1.0:1789466103.530195:0:223621:0:(import.c:2262:obd_at_measure()) AT ffff8967e7ba6c18 change: old=5 new=10 delta=5 (val=10) hist 10 1 0 0 00000100:00080000:1.0:1789466103.530198:0:223621:0:(import.c:630:import_select_connection()) MGC192.168.203.142@tcp: tried all connections, increasing latency to 5s 00000100:00000040:1.0:1789466103.530201:0:223621:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:1.0:1789466103.530204:0:223621:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1789466103.530206:0:223621:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:1.0:1789466103.530208:0:223621:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000001:1.0:1789466103.530215:0:223621:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1789466103.530217:0:223621:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b794228 00000020:00000001:1.0:1789466103.530220:0:223621:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1789466103.530223:0:223621:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967e7ba1800 refcount=3 00000020:00000001:1.0:1789466103.530225:0:223621:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678095210496 : -130395614341120 : ffff8967e7ba1800) 00000020:00000001:1.0:1789466103.530228:0:223621:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678095210496 : -130395614341120 : ffff8967e7ba1800) 00000100:00000040:1.0:1789466103.530230:0:223621:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:1.0:1789466103.530232:0:223621:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1789466103.530234:0:223621:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:1.0:1789466103.530236:0:223621:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000040:1.0:1789466103.530239:0:223621:0:(genops.c:879:class_export_put()) PUTting export ffff8967e7ba1800 : new refcount 2 00000100:00080000:1.0:1789466103.530242:0:223621:0:(import.c:666:import_select_connection()) MGC192.168.203.142@tcp: import ffff8967e7ba6800 using connection 192.168.203.142@tcp/0@lo 00000100:00000001:1.0:1789466103.530245:0:223621:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466103.530247:0:223621:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1789466103.530250:0:223621:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466103.530253:0:223621:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530256:0:223621:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1789466103.530258:0:223621:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1789466103.530262:0:223621:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8967d3738a80. 00000020:00000040:1.0:1789466103.530265:0:223621:0:(genops.c:1209:class_import_get()) import ffff8967e7ba6800 refcount=4 obd=MGC192.168.203.142@tcp 02000000:00000001:1.0:1789466103.530269:0:223621:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1789466103.530270:0:223621:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530273:0:223621:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789466103.530281:0:223621:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789466103.530285:0:223621:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789466103.530292:0:223621:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8967fe7f2000. 00000100:00000001:1.0:1789466103.530298:0:223621:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466103.530303:0:223621:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789466103.530311:0:223621:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789466103.530315:0:223621:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466103.530319:0:223621:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a187de8] to pc [ptlrpcd_rcv+-1] req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530333:0:223621:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1789466103.530336:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466103.530340:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466103.530343:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466103.530346:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466103.530349:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466103.530352:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466103.530354:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466103.530357:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1789466103.530362:0:223621:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18356) 00000100:00000001:1.0:1789466103.530373:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466103.530374:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466103.530376:0:117762:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789466103.530380:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530387:0:117762:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789466103.530389:0:117762:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466103.530401:0:117762:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789466103.530403:0:117762:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466103.530406:0:117762:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466103.530409:0:117762:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8967d3738a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:4861e2df-2452-4fd5-a0b1-355ea409d38c:117762:1876386404768768:0@lo:250:kworker.0 00000100:00000001:1.0:1789466103.530413:0:117762:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789466103.530417:0:117762:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789466103.530419:0:117762:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466103.530421:0:117762:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789466103.530424:0:117762:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8967fe7f0c00. 02000000:00000001:1.0:1789466103.530426:0:117762:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1789466103.530430:0:117762:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8967f4dbfc08. 00000400:00000010:1.0:1789466103.530435:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351198. 00000100:00000200:1.0:1789466103.530440:0:117762:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1876386404768768, portal 25 00000100:00000001:1.0:1789466103.530443:0:117762:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789466103.530444:0:117762:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613677755042432 : -130395954509184 : ffff8967d3738a80) 00000100:00000040:1.0:1789466103.530449:0:117762:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 1789466118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530455:0:117762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466103.530458:0:117762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1789466103.530461:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351550. 00000100:00000200:1.0:1789466103.530464:0:117762:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1876386404768768, offset 0 00000400:00000200:1.0:1789466103.530470:0:117762:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1789466103.530482:0:117762:0:(lib-move.c:4645:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1789466103.530492:0:117762:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 520 into portal 26 MB=0x6aa90411e1c00 00000400:00000100:1.0:1789466103.530498:0:117762:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1876386404768768 offset 0 length 520: 4 00000400:00000200:1.0:1789466103.530503:0:117762:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8966e263dfe0 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1789466103.530505:0:117762:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:1.0:1789466103.530515:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351550 00000400:00000010:1.0:1789466103.530517:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351550. 00000100:00000001:1.0:1789466103.530520:0:117762:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1789466103.530524:0:117762:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 1789466118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530531:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466103.530540:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 1789466118 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466103.530565:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530567:0:117762:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1789466103.530574:0:117762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530578:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 0 is 4313024114 00000100:00000010:1.0:1789466103.530583:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc11f60. 00000100:00000001:1.0:1789466103.530589:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1789466103.530599:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 1 is 4313024114 00000100:00000010:1.0:1789466103.530601:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc114e0. 00000100:00000001:1.0:1789466103.530605:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1789466103.530608:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 2 is 4313024114 00000100:00000010:1.0:1789466103.530610:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc11de0. 00000100:00000001:1.0:1789466103.530613:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1789466103.530615:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 3 is 4313024114 00000100:00000010:1.0:1789466103.530618:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc116c0. 00000100:00000001:1.0:1789466103.530621:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1789466103.530624:0:117762:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789466103.530627:0:117762:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530630:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530632:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.530636:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466103.530638:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.530640:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466103.530641:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466103.530642:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466103.530643:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466103.783211:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466103.783220:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466103.783247:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466103.783250:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466103.783253:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:2.0:1789466103.942455:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.942621:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466103.942628:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.942640:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466103.942656:0:117758:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466103.944908:0:117758:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94780 00000400:00000200:2.0:1789466103.944926:0:117758:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 5152 00000800:00000001:2.0:1789466103.944939:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.945356:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466103.945362:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466103.945369:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466103.945381:0:117758:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466103.945383:0:117758:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466103.945395:0:117758:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189ed80. 00000100:00000040:2.0:1789466103.945470:0:117758:0:(events.c:341:request_in_callback()) incoming req@ffff89680189ed80 x1876386402355072 msgsize 224 00000100:00100000:2.0:1789466103.945476:0:117758:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466103.945566:0:117758:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466103.945583:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.945588:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466103.945591:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.945593:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466103.945601:0:117758:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466103.945617:0:117758:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94800 00000400:00000200:2.0:1789466103.945623:0:117758:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 5376 00000800:00000001:2.0:1789466103.945628:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.945652:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466103.945654:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466103.945658:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466103.945662:0:117758:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466103.945664:0:117758:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466103.945668:0:117758:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189d180. 00000100:00000040:2.0:1789466103.945671:0:117758:0:(events.c:341:request_in_callback()) incoming req@ffff89680189d180 x1876386402355200 msgsize 224 00000100:00100000:2.0:1789466103.945675:0:117758:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:0.0:1789466103.945772:0:170896:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1789466103.945812:0:117758:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466103.945819:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466103.945822:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466103.945971:0:201485:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1789466103.946164:0:170896:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402355072 02000000:00000001:0.0:1789466103.946169:0:170896:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1789466103.946171:0:170896:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789466103.946175:0:170896:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466103.946181:0:170896:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466103.946187:0:170896:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402355072 00000020:00000001:0.0:1789466103.946192:0:170896:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466103.946194:0:170896:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b86f3e6 00000020:00000001:0.0:1789466103.946197:0:170896:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466103.946202:0:170896:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967c3b5d800 refcount=7 00000020:00000001:0.0:1789466103.946205:0:170896:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000020:00000001:0.0:1789466103.946208:0:170896:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000100:00000001:0.0:1789466103.946214:0:170896:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466103.946221:0:170896:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e940. 00000020:00000010:0.0:1789466103.946227:0:170896:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ee05e880. 00000020:00000040:0.0:1789466103.946235:0:170896:0:(obd_config.c:995:class_incref()) incref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000001:0.0:1789466103.946248:0:170896:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:0.0:1789466103.946257:0:170896:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967d2e7c000. 00000020:00000010:0.0:1789466103.946264:0:170896:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3528d80. 00000020:00000010:0.0:1789466103.946271:0:170896:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967e94d8d48. 00000100:00000040:0.0:1789466103.946286:0:170896:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00000001:0.0:1789466103.946289:0:170896:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1789466103.946291:0:170896:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1789466103.946295:0:170896:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466103.946305:0:170896:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1789466103.946308:0:170896:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1789466103.946310:0:170896:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466103.946316:0:170896:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466103.946329:0:170896:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1789466103.946330:0:170896:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402355200 02000000:00000001:0.0:1789466103.946332:0:170896:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1789466103.946333:0:170896:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789466103.946335:0:170896:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466103.946338:0:170896:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466103.946350:0:170896:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402355200 00000020:00000001:0.0:1789466103.946352:0:170896:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466103.946354:0:170896:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b8705ac 00000020:00000001:0.0:1789466103.946355:0:170896:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466103.946358:0:170896:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5cc5000 refcount=7 00000020:00000001:0.0:1789466103.946361:0:170896:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000020:00000001:0.0:1789466103.946363:0:170896:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000100:00000001:0.0:1789466103.946367:0:170896:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466103.946369:0:170896:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e880. 00000020:00000010:0.0:1789466103.946373:0:170896:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ede8e440. 00000020:00000040:0.0:1789466103.946378:0:170896:0:(obd_config.c:995:class_incref()) incref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:0.0:1789466103.946381:0:170896:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:0.0:1789466103.946386:0:170896:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967d2e7dc00. 00000020:00000010:0.0:1789466103.946389:0:170896:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3528d00. 00000020:00000010:0.0:1789466103.946393:0:170896:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967e94d8898. 00000100:00000001:0.0:1789466103.946397:0:170896:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1789466103.946408:0:170896:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1789466103.946410:0:170896:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466103.946414:0:170896:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1789466103.946416:0:170896:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1789466103.946418:0:170896:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466103.946428:0:170896:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466103.946438:0:170896:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1789466103.946441:0:170896:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00080000:3.0:1789466103.946444:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1789466103.946448:0:170896:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3503 00000020:00000040:3.0:1789466103.946452:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000040:0.0:1789466103.946455:0:170896:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967c3b5d800 : new rpc_count 1 00000100:00080000:3.0:1789466103.946459:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00000001:0.0:1789466103.946460:0:170896:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528261504 : -130395181290112 : ffff89680189ed80) 00000020:00000040:3.0:1789466103.946464:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000040:0.0:1789466103.946474:0:170896:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189ed80 x1876386402355072/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:599/0 lens 224/0 e 0 to 0 dl 1789466114 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789466103.946486:0:170896:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466103.946489:0:170896:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e180. 00000100:00000001:0.0:1789466103.946492:0:170896:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1789466103.946495:0:170896:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967ee05e180. 00000100:00100000:0.0:1789466103.946507:0:170896:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402355072:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:0.0:1789466103.946512:0:170896:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402355072 00000020:00000001:0.0:1789466103.946516:0:170896:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1789466103.946521:0:170896:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1789466103.946524:0:170896:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466103.946527:0:170896:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1789466103.946530:0:170896:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:0.0:1789466103.946534:0:170896:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1789466103.946537:0:170896:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1789466103.946550:0:170896:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1789466103.946553:0:170896:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1789466103.946556:0:170896:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466103.946559:0:170896:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1789466103.946564:0:170896:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1789466103.946567:0:170896:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1789466103.946574:0:170896:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967ee8fc000. 02000000:00000001:0.0:1789466103.946577:0:170896:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466103.946580:0:170896:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466103.946582:0:170896:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1789466103.946586:0:170896:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 17179870928, transno 0, xid 1876386402355072 00010000:00000001:0.0:1789466103.946589:0:170896:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:0.0:1789466103.946595:0:170896:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189ed80 x1876386402355072/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:599/0 lens 224/224 e 0 to 0 dl 1789466114 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789466103.946990:0:121172:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1789466103.946993:0:121172:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1789466103.946999:0:121172:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3504 00000100:00000040:2.0:1789466103.947002:0:121172:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967f5cc5000 : new rpc_count 1 00000100:00000001:2.0:1789466103.947004:0:121172:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528254336 : -130395181297280 : ffff89680189d180) 00000100:00000040:2.0:1789466103.947011:0:121172:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189d180 x1876386402355200/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:599/0 lens 224/0 e 0 to 0 dl 1789466114 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789466103.947019:0:121172:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1789466103.947024:0:121172:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ede8e8c0. 00000100:00000001:2.0:1789466103.947027:0:121172:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1789466103.947029:0:121172:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967ede8e8c0. 00000100:00100000:2.0:1789466103.947034:0:121172:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402355200:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:2.0:1789466103.947066:0:121172:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402355200 00000020:00000001:2.0:1789466103.947069:0:121172:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1789466103.947071:0:121172:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1789466103.947073:0:121172:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466103.947075:0:121172:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1789466103.947078:0:121172:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:2.0:1789466103.947080:0:121172:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1789466103.947082:0:121172:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1789466103.947084:0:121172:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1789466103.947085:0:121172:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1789466103.947087:0:121172:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466103.947089:0:121172:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1789466103.947091:0:121172:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1789466103.947093:0:121172:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1789466103.947097:0:121172:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c641dc00. 02000000:00000001:2.0:1789466103.947099:0:121172:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789466103.947101:0:121172:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466103.947103:0:121172:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1789466103.947106:0:121172:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 12884913986, transno 0, xid 1876386402355200 00010000:00000001:2.0:1789466103.947108:0:121172:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:2.0:1789466103.947112:0:121172:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189d180 x1876386402355200/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:599/0 lens 224/224 e 0 to 0 dl 1789466114 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1789466103.947120:0:121172:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1789466103.947124:0:121172:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789466103.947129:0:121172:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=12 v=5 (1 1 1 1) 00000100:00000001:2.0:1789466103.947135:0:121172:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1789466103.947138:0:121172:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:2.0:1789466103.947141:0:121172:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:2.0:1789466103.947144:0:121172:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1789466103.947146:0:121172:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789466103.947151:0:121172:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789466103.947154:0:121172:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:2.0:1789466103.947160:0:121172:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed964440. 00000100:00000200:2.0:1789466103.947168:0:121172:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402355200, offset 224 00000400:00000200:2.0:1789466103.947183:0:121172:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:2.0:1789466103.947201:0:121172:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:2.0:1789466103.947209:0:121172:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93555:93555:256:4294967295] 192.168.203.42@tcp LPNI seq info [93551:93551:8:4294967295] 00000400:00000200:2.0:1789466103.947221:0:121172:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:2.0:1789466103.947228:0:121172:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:2.0:1789466103.947235:0:121172:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8967fe8e0800. 00000800:00000200:2.0:1789466103.947243:0:121172:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:2.0:1789466103.947252:0:121172:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:2.0:1789466103.947257:0:121172:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8967fe8e0800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789466103.947278:0:121172:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789466103.947282:0:121172:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:2.0:1789466103.947284:0:121172:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:2.0:1789466103.947287:0:121172:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466103.947289:0:121172:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1789466103.947294:0:121172:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189d180 x1876386402355200/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:599/0 lens 224/224 e 0 to 0 dl 1789466114 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789466103.947304:0:121172:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402355200:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 283us (1631us total) trans 0 rc 0/0 00000100:00100000:2.0:1789466103.947313:0:121172:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3504 00000100:00000040:2.0:1789466103.947317:0:121172:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967f5cc5000 : new rpc_count 0 00000100:00000001:2.0:1789466103.947320:0:121172:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1789466103.947321:0:121172:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1789466103.947327:0:121172:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3528d00. 00000020:00000010:2.0:1789466103.947332:0:121172:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967e94d8898. 00000020:00000010:2.0:1789466103.947337:0:121172:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967d2e7dc00. 00000020:00000040:2.0:1789466103.947342:0:121172:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5cc5000 : new refcount 6 00000100:00000001:2.0:1789466103.947347:0:121172:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1789466103.947872:0:117756:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789466103.947878:0:117756:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8967fe8e0800. 00000400:00000200:3.0:1789466103.947884:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466103.947897:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:3.0:1789466103.947904:0:117756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed964440 00000400:00000010:3.0:1789466103.947908:0:117756:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed964440. 00000100:00000001:3.0:1789466103.947913:0:117756:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:3.0:1789466103.947915:0:117756:0:(events.c:402:reply_out_callback()) Process leaving 00010000:00000001:0.0:1789466103.948282:0:170896:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1789466103.948289:0:170896:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789466103.948294:0:170896:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=12 v=5 (1 1 1 1) 00000100:00000001:0.0:1789466103.948300:0:170896:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466103.948303:0:170896:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:0.0:1789466103.948305:0:170896:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:0.0:1789466103.948308:0:170896:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1789466103.948311:0:170896:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466103.948313:0:170896:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789466103.948316:0:170896:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:0.0:1789466103.948321:0:170896:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967f7b93e58. 00000100:00000200:0.0:1789466103.948335:0:170896:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402355072, offset 224 00000400:00000200:0.0:1789466103.948342:0:170896:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:0.0:1789466103.948352:0:170896:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:0.0:1789466103.948357:0:170896:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93556:93556:256:4294967295] 192.168.203.42@tcp LPNI seq info [93552:93552:8:4294967295] 00000400:00000200:0.0:1789466103.948367:0:170896:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:0.0:1789466103.948372:0:170896:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:0.0:1789466103.948376:0:170896:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8967c9f9e400. 00000800:00000200:0.0:1789466103.948382:0:170896:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:0.0:1789466103.948387:0:170896:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:0.0:1789466103.948392:0:170896:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8967c9f9e400 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789466103.948412:0:170896:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466103.948416:0:170896:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:0.0:1789466103.948419:0:170896:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:0.0:1789466103.948422:0:170896:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466103.948425:0:170896:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1789466103.948432:0:170896:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189ed80 x1876386402355072/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:599/0 lens 224/224 e 0 to 0 dl 1789466114 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789466103.948443:0:170896:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402355072:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 1956us (2972us total) trans 0 rc 0/0 00000100:00100000:0.0:1789466103.948452:0:170896:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3503 00000100:00000040:0.0:1789466103.948455:0:170896:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967c3b5d800 : new rpc_count 0 00000100:00000001:0.0:1789466103.948457:0:170896:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1789466103.948459:0:170896:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1789466103.948464:0:170896:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3528d80. 00000020:00000010:0.0:1789466103.948468:0:170896:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967e94d8d48. 00000020:00000010:0.0:1789466103.948471:0:170896:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967d2e7c000. 00000020:00000040:0.0:1789466103.948476:0:170896:0:(genops.c:879:class_export_put()) PUTting export ffff8967c3b5d800 : new refcount 6 00000100:00000001:0.0:1789466103.948480:0:170896:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1789466103.948482:0:117757:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789466103.948487:0:117757:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8967c9f9e400. 00000400:00000200:3.0:1789466103.948493:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466103.948499:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:3.0:1789466103.948503:0:117757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967f7b93e58 00000400:00000010:3.0:1789466103.948505:0:117757:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967f7b93e58. 00000100:00000001:3.0:1789466103.948509:0:117757:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:3.0:1789466103.948511:0:117757:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00000001:1.0:1789466103.948801:0:201485:0:(service.c:2287:ptlrpc_server_handle_req_in()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789466104.019486:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466104.019514:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466104.019519:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466104.019553:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466104.019568:0:117758:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466104.019592:0:117758:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 520 into portal 26 MB=0x6aa9040f94880 00000400:00000100:2.0:1789466104.019603:0:117758:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-192.168.203.42@tcp portal 26 match 1876386402355328 offset 0 length 520: 4 00000400:00000200:2.0:1789466104.019609:0:117758:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8967e7afa420 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:2.0:1789466104.019612:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1789466104.019628:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466104.019640:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466104.019642:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466104.019645:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466104.551481:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466104.551501:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466104.551503:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466104.551510:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466104.551513:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466104.551551:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466104.551554:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466104.809028:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466104.809073:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466104.809238:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466104.809243:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466104.809246:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00080000:00000010:2.0:1789466104.954189:0:224824:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967e983d480. 00080000:00000010:2.0:1789466104.954204:0:224824:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967e983d480. 00000020:00000001:2.0:1789466104.954212:0:224824:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:2.0:1789466104.954222:0:224824:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967e983d480. 00080000:00000010:2.0:1789466104.954228:0:224824:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967e983d480. 00000020:00000001:2.0:1789466104.954232:0:224824:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1789466105.312226:0:224861:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:2.0:1789466105.312236:0:224861:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity test 160j: client can be umounted while its chanangelog is being used ========================================================== 05:55:03 (1789466103) 00000001:00000001:2.0:1789466105.325858:0:224861:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00000100:00000001:1.0:1789466105.575219:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466105.575244:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466105.575247:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466105.575256:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466105.575260:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466105.575269:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466105.575273:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466105.833734:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466105.833744:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466105.834060:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789466105.834064:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789466105.834068:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466106.599487:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466106.599506:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466106.599509:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466106.599516:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466106.599518:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466106.599534:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466106.599538:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466106.855244:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466106.855260:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466106.855881:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789466106.855887:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789466106.855892:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466107.624206:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466107.624224:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466107.624226:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466107.624232:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466107.624235:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466107.624240:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466107.624243:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466107.879215:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466107.879224:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466107.879261:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466107.879265:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466107.879268:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789466108.583596:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.583612:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.583616:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.583620:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.583623:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.583627:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.583630:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.583633:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.583637:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.583640:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1789466108.583646:0:134719:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18361) 00000100:00000001:1.0:1789466108.647206:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466108.647219:0:117762:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1789466108.647240:0:117762:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789466088/real 1789466088] req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1789466108.647265:0:117762:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789466108.647268:0:117762:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789466108.647272:0:117762:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8967f4dbfe70. 00000400:00000200:1.0:1789466108.647281:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351aa0 00000400:00000010:1.0:1789466108.647284:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351aa0. 00000100:00000001:1.0:1789466108.647291:0:117762:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789466108.647295:0:117762:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789466108.647338:0:117762:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647347:0:117762:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789466108.647350:0:117762:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647353:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.647356:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1789466108.647360:0:117762:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647367:0:117762:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647371:0:117762:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1789466108.647375:0:117762:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789466088/real 1789466088] req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1789466108.647384:0:117762:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789466108.647385:0:117762:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789466108.647388:0:117762:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8967f4dbf0b0. 00000400:00000200:1.0:1789466108.647391:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c63513b8 00000400:00000010:1.0:1789466108.647393:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c63513b8. 00000100:00000001:1.0:1789466108.647395:0:117762:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789466108.647398:0:117762:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789466108.647405:0:117762:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647410:0:117762:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789466108.647412:0:117762:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647414:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.647416:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1789466108.647418:0:117762:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647423:0:117762:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647425:0:117762:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1789466108.647437:0:117762:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789466088/real 1789466088] req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1789466108.647444:0:117762:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789466108.647445:0:117762:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789466108.647447:0:117762:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8967f4dbf630. 00000400:00000200:1.0:1789466108.647450:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351330 00000400:00000010:1.0:1789466108.647451:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351330. 00000100:00000001:1.0:1789466108.647453:0:117762:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789466108.647456:0:117762:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789466108.647462:0:117762:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647466:0:117762:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789466108.647468:0:117762:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647469:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.647470:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1789466108.647473:0:117762:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647479:0:117762:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647481:0:117762:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1789466108.647485:0:117762:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789466088/real 1789466088] req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1789466108.647491:0:117762:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789466108.647492:0:117762:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789466108.647494:0:117762:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8967f4dbf7e8. 00000400:00000200:1.0:1789466108.647496:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351cc0 00000400:00000010:1.0:1789466108.647497:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351cc0. 00000100:00000001:1.0:1789466108.647499:0:117762:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789466108.647502:0:117762:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789466108.647522:0:117762:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647535:0:117762:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789466108.647537:0:117762:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647538:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.647540:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1789466108.647544:0:117762:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647550:0:117762:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647561:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466108.647564:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647568:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466108.647572:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466108.647575:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789466108.647580:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647588:0:117762:0:(client.c:2143:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1789466108.647591:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647592:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.647593:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647596:0:117762:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1789466108.647599:0:117762:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1789466108.647603:0:117762:0:(import.c:65:import_set_state_nolock()) ffff8966c3df3800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789466108.647696:0:117762:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-110) 00000100:00000001:1.0:1789466108.647700:0:117762:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1789466108.647704:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466108.647712:0:117762:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89680189dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:117762:1876386404768256:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.647717:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789466108.647720:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.647737:0:117762:0:(client.c:2143:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1789466108.647739:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647740:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.647741:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.647743:0:117762:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1789466108.647745:0:117762:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1789466108.647747:0:117762:0:(import.c:65:import_set_state_nolock()) ffff8966c3df5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789466108.650607:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.650622:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.650626:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.650630:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.650659:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.650662:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.650666:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.650670:0:134719:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1789466108.650673:0:134719:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1789466108.650677:0:134719:0:(import.c:65:import_set_state_nolock()) ffff8966c3df3800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1789466108.650683:0:134719:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1789466108.650689:0:134719:0:(import.c:545:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 18336 00000100:00001000:0.0:1789466108.650697:0:134719:0:(import.c:2196:obd_at_measure()) add 20 to ffff8966c3df3c18 time=142 v=15 (15 1 0 0) 00000100:00001000:0.0:1789466108.650700:0:134719:0:(import.c:2262:obd_at_measure()) AT ffff8966c3df3c18 change: old=15 new=20 delta=5 (val=20) hist 20 1 0 0 00000100:00080000:0.0:1789466108.650704:0:134719:0:(import.c:630:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00000040:0.0:1789466108.650707:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.650710:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.650712:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.650714:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000001:0.0:1789466108.650726:0:134719:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466108.650729:0:134719:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b794730 00000020:00000001:0.0:1789466108.650732:0:134719:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466108.650736:0:134719:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8966c3df3000 refcount=8 00000020:00000001:0.0:1789466108.650738:0:134719:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613673198694400 : -130400510857216 : ffff8966c3df3000) 00000020:00000001:0.0:1789466108.650741:0:134719:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613673198694400 : -130400510857216 : ffff8966c3df3000) 00000100:00000040:0.0:1789466108.650743:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.650745:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.650746:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.650748:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000040:0.0:1789466108.650751:0:134719:0:(genops.c:879:class_export_put()) PUTting export ffff8966c3df3000 : new refcount 7 00000100:00080000:0.0:1789466108.650755:0:134719:0:(import.c:666:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8966c3df3800 using connection 192.168.203.142@tcp/0@lo 00000100:00000001:0.0:1789466108.650758:0:134719:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466108.650768:0:134719:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1789466108.650777:0:134719:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789466108.650780:0:134719:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.650783:0:134719:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1789466108.650786:0:134719:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1789466108.650795:0:134719:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8967cb942a00. 00000020:00000040:0.0:1789466108.650799:0:134719:0:(genops.c:1209:class_import_get()) import ffff8966c3df3800 refcount=4 obd=lustre-MDT0000-lwp-OST0001 02000000:00000001:0.0:1789466108.650804:0:134719:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1789466108.650806:0:134719:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.650810:0:134719:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1789466108.650813:0:134719:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1789466108.650818:0:134719:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1789466108.650829:0:134719:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8967ee8fd400. 00000100:00000001:0.0:1789466108.650837:0:134719:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466108.650848:0:134719:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8967cb942a00 x1876386404768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1789466108.650859:0:134719:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1789466108.650864:0:134719:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466108.650869:0:134719:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000969a4da] to pc [ptlrpcd_rcv+-1] req@ffff8967cb942a00 x1876386404768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789466108.650883:0:134719:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1789466108.650886:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.650888:0:134719:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1789466108.650890:0:134719:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1789466108.650891:0:134719:0:(import.c:65:import_set_state_nolock()) ffff8966c3df5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1789466108.650894:0:134719:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1789466108.650896:0:134719:0:(import.c:545:import_select_connection()) lustre-MDT0001-lwp-OST0001: connect to NID 0@lo last attempt 18336 00000100:00001000:0.0:1789466108.650898:0:134719:0:(import.c:2196:obd_at_measure()) add 20 to ffff8966c3df5c18 time=158 v=15 (15 1 1 1) 00000100:00001000:0.0:1789466108.650901:0:134719:0:(import.c:2262:obd_at_measure()) AT ffff8966c3df5c18 change: old=15 new=20 delta=5 (val=20) hist 20 15 1 1 00000100:00080000:0.0:1789466108.650903:0:134719:0:(import.c:630:import_select_connection()) lustre-MDT0001-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00000040:0.0:1789466108.650905:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.650907:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.650916:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.650918:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000001:0.0:1789466108.650921:0:134719:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466108.650923:0:134719:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b794745 00000020:00000001:0.0:1789466108.650924:0:134719:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466108.650927:0:134719:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5d04800 refcount=5 00000020:00000001:0.0:1789466108.650929:0:134719:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331545600 : -130395378006016 : ffff8967f5d04800) 00000020:00000001:0.0:1789466108.650931:0:134719:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331545600 : -130395378006016 : ffff8967f5d04800) 00000100:00000040:0.0:1789466108.650933:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.650935:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.650936:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.650937:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000040:0.0:1789466108.650939:0:134719:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5d04800 : new refcount 4 00000100:00080000:0.0:1789466108.650942:0:134719:0:(import.c:666:import_select_connection()) lustre-MDT0001-lwp-OST0001: import ffff8966c3df5800 using connection 192.168.203.142@tcp/0@lo 00000100:00000001:0.0:1789466108.650944:0:134719:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466108.650946:0:134719:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1789466108.650948:0:134719:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789466108.650951:0:134719:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.650953:0:134719:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1789466108.650954:0:134719:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1789466108.650957:0:134719:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8967cb941f80. 00000020:00000040:0.0:1789466108.650960:0:134719:0:(genops.c:1209:class_import_get()) import ffff8966c3df5800 refcount=4 obd=lustre-MDT0001-lwp-OST0001 02000000:00000001:0.0:1789466108.650962:0:134719:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1789466108.650964:0:134719:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.650965:0:134719:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1789466108.650967:0:134719:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1789466108.650969:0:134719:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1789466108.650972:0:134719:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8967ee8fd800. 00000100:00000001:0.0:1789466108.650976:0:134719:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466108.650981:0:134719:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8967cb941f80 x1876386404769024/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1789466108.650988:0:134719:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1789466108.650990:0:134719:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466108.650999:0:134719:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b389be71] to pc [ptlrpcd_rcv+-1] req@ffff8967cb941f80 x1876386404769024/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789466108.651008:0:134719:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466108.651013:0:134719:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18361) 00000100:00080000:1.0:1789466108.651363:0:117762:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 0@lo failed (-110) 00000100:00000001:1.0:1789466108.651367:0:117762:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1789466108.651375:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466108.651384:0:117762:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89680189ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-lwp-OST0001_UUID:117762:1876386404768384:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.651405:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789466108.651409:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651416:0:117762:0:(client.c:2143:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1789466108.651418:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651421:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.651423:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651425:0:117762:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1789466108.651427:0:117762:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1789466108.651431:0:117762:0:(import.c:65:import_set_state_nolock()) ffff8966c6123800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789466108.651436:0:117762:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 0@lo failed (-110) 00000100:00000001:1.0:1789466108.651438:0:117762:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1789466108.651441:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466108.651460:0:117762:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89680189c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-lwp-OST0000_UUID:117762:1876386404768128:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.651465:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789466108.651468:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651474:0:117762:0:(client.c:2143:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1789466108.651476:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651478:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466108.651480:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651481:0:117762:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1789466108.651483:0:117762:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1789466108.651485:0:117762:0:(import.c:65:import_set_state_nolock()) ffff8966c6120000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789466108.651490:0:117762:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-110) 00000100:00000001:1.0:1789466108.651492:0:117762:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1789466108.651495:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466108.651502:0:117762:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89680189d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:117762:1876386404768000:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.651516:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.651520:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.651524:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff89680189dc00 x1876386404768256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651531:0:117762:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789466108.651534:0:117762:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789466108.651537:0:117762:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8967fe7f0000. 02000000:00000001:1.0:1789466108.651544:0:117762:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789466108.651546:0:117762:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789466108.651548:0:117762:0:(genops.c:1222:class_import_put()) import ffff8966c3df3800 refcount=3 obd=lustre-MDT0000-lwp-OST0001 00000020:00000001:1.0:1789466108.651550:0:117762:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789466108.651558:0:117762:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8967c641cc00. 02000000:00000001:1.0:1789466108.651563:0:117762:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789466108.651565:0:117762:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789466108.651568:0:117762:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89680189dc00. 00000100:00000001:1.0:1789466108.651572:0:117762:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789466108.651574:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651576:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.651579:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff89680189ea00 x1876386404768384/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651585:0:117762:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789466108.651587:0:117762:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789466108.651588:0:117762:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8967fe7f1400. 02000000:00000001:1.0:1789466108.651592:0:117762:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789466108.651594:0:117762:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789466108.651595:0:117762:0:(genops.c:1222:class_import_put()) import ffff8966c3df5800 refcount=3 obd=lustre-MDT0001-lwp-OST0001 00000020:00000001:1.0:1789466108.651597:0:117762:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789466108.651598:0:117762:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8967c641e800. 02000000:00000001:1.0:1789466108.651601:0:117762:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789466108.651602:0:117762:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789466108.651603:0:117762:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89680189ea00. 00000100:00000001:1.0:1789466108.651608:0:117762:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789466108.651609:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651611:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.651614:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff89680189c700 x1876386404768128/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651621:0:117762:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789466108.651622:0:117762:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789466108.651623:0:117762:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8967fe7f2800. 02000000:00000001:1.0:1789466108.651626:0:117762:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789466108.651627:0:117762:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789466108.651628:0:117762:0:(genops.c:1222:class_import_put()) import ffff8966c6123800 refcount=2 obd=lustre-MDT0001-lwp-OST0000 00000020:00000001:1.0:1789466108.651630:0:117762:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789466108.651632:0:117762:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8967c641ec00. 02000000:00000001:1.0:1789466108.651634:0:117762:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789466108.651635:0:117762:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789466108.651636:0:117762:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89680189c700. 00000100:00000001:1.0:1789466108.651639:0:117762:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789466108.651645:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651647:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.651651:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff89680189d500 x1876386404768000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 1 dl 1789466108 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651657:0:117762:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789466108.651659:0:117762:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789466108.651661:0:117762:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8967fe7f1800. 02000000:00000001:1.0:1789466108.651663:0:117762:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789466108.651664:0:117762:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789466108.651665:0:117762:0:(genops.c:1222:class_import_put()) import ffff8966c6120000 refcount=2 obd=lustre-MDT0000-lwp-OST0000 00000020:00000001:1.0:1789466108.651667:0:117762:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789466108.651669:0:117762:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8967c641f000. 02000000:00000001:1.0:1789466108.651671:0:117762:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789466108.651672:0:117762:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789466108.651673:0:117762:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89680189d500. 00000100:00000001:1.0:1789466108.651675:0:117762:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789466108.651676:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651679:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651689:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466108.651692:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.651694:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466108.651697:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466108.651699:0:117762:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789466108.651702:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8967cb942a00 x1876386404768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651708:0:117762:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789466108.651711:0:117762:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.651714:0:117762:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789466108.651717:0:117762:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466108.651720:0:117762:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466108.651723:0:117762:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8967cb942a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:117762:1876386404768896:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.651727:0:117762:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789466108.651733:0:117762:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789466108.651735:0:117762:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.651738:0:117762:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789466108.651743:0:117762:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8967fe7f1800. 02000000:00000001:1.0:1789466108.651750:0:117762:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1789466108.651758:0:117762:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8967f4dbf7e8. 00000400:00000010:1.0:1789466108.651765:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351cc0. 00000100:00000200:1.0:1789466108.651774:0:117762:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1876386404768896, portal 10 00000100:00000001:1.0:1789466108.651777:0:117762:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789466108.651779:0:117762:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613677622962688 : -130396086588928 : ffff8967cb942a00) 00000100:00000040:1.0:1789466108.651787:0:117762:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8967cb942a00 x1876386404768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651794:0:117762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466108.651797:0:117762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1789466108.651800:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351330. 00000100:00000200:1.0:1789466108.651803:0:117762:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1876386404768896, offset 0 00000400:00000200:1.0:1789466108.651811:0:117762:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1789466108.651829:0:117762:0:(lib-move.c:4645:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1789466108.651844:0:117762:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 520 into portal 12 MB=0x6aa90411e1c80 00000400:00000100:1.0:1789466108.651851:0:117762:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 12 match 1876386404768896 offset 0 length 520: 4 00000400:00000200:1.0:1789466108.651857:0:117762:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8966e263dfe0 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1789466108.651860:0:117762:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:1.0:1789466108.651874:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351330 00000400:00000010:1.0:1789466108.651878:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351330. 00000100:00000001:1.0:1789466108.651882:0:117762:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1789466108.651887:0:117762:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8967cb942a00 x1876386404768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651895:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.651897:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8967cb942a00 x1876386404768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651903:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.651905:0:117762:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1789466108.651910:0:117762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466108.651916:0:117762:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: using a timeout of 18711 usecs (19 jiffies) 00000100:00000001:1.0:1789466108.651920:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS new deadline estimation for cpu 0 is 4313024087 00000100:00000001:1.0:1789466108.651923:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS request active for cpu 0, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.651931:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS new deadline estimation for cpu 1 is 4313024087 00000100:00000001:1.0:1789466108.651934:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS request active for cpu 1, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.651935:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS new deadline estimation for cpu 2 is 4313024087 00000100:00000001:1.0:1789466108.651938:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS request active for cpu 2, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.651939:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS new deadline estimation for cpu 3 is 4313024087 00000100:00000001:1.0:1789466108.651942:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0001: PM QoS request active for cpu 3, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.651944:0:117762:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789466108.651947:0:117762:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.651949:0:117762:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789466108.651953:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8967cb941f80 x1876386404769024/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.651959:0:117762:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789466108.651960:0:117762:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.651963:0:117762:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789466108.651965:0:117762:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466108.651966:0:117762:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466108.651969:0:117762:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8967cb941f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-lwp-OST0001_UUID:117762:1876386404769024:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.651972:0:117762:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789466108.651974:0:117762:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789466108.651975:0:117762:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.651976:0:117762:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789466108.651979:0:117762:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8967fe7f2800. 02000000:00000001:1.0:1789466108.651981:0:117762:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1789466108.651984:0:117762:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8967f4dbf630. 00000400:00000010:1.0:1789466108.651988:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351330. 00000100:00000200:1.0:1789466108.651991:0:117762:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1876386404769024, portal 10 00000100:00000001:1.0:1789466108.651993:0:117762:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789466108.651994:0:117762:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613677622960000 : -130396086591616 : ffff8967cb941f80) 00000100:00000040:1.0:1789466108.652000:0:117762:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8967cb941f80 x1876386404769024/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.652005:0:117762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466108.652013:0:117762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1789466108.652015:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c63513b8. 00000100:00000200:1.0:1789466108.652017:0:117762:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1876386404769024, offset 0 00000400:00000200:1.0:1789466108.652021:0:117762:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1789466108.652026:0:117762:0:(lib-move.c:4645:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1789466108.652031:0:117762:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 520 into portal 12 MB=0x6aa90411e1d00 00000400:00000100:1.0:1789466108.652034:0:117762:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 12 match 1876386404769024 offset 0 length 520: 4 00000400:00000200:1.0:1789466108.652061:0:117762:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8966e263dfe0 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1789466108.652064:0:117762:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:1.0:1789466108.652068:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c63513b8 00000400:00000010:1.0:1789466108.652070:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c63513b8. 00000100:00000001:1.0:1789466108.652072:0:117762:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1789466108.652076:0:117762:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8967cb941f80 x1876386404769024/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.652084:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.652086:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8967cb941f80 x1876386404769024/t0(0) o38->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.652091:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.652092:0:117762:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1789466108.652095:0:117762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466108.652098:0:117762:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: using a timeout of 19415 usecs (20 jiffies) 00000100:00000001:1.0:1789466108.652100:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS new deadline estimation for cpu 0 is 4313024089 00000100:00000001:1.0:1789466108.652102:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS request active for cpu 0, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.652104:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS new deadline estimation for cpu 1 is 4313024089 00000100:00000001:1.0:1789466108.652105:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS request active for cpu 1, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.652107:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS new deadline estimation for cpu 2 is 4313024089 00000100:00000001:1.0:1789466108.652109:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS request active for cpu 2, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.652110:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS new deadline estimation for cpu 3 is 4313024089 00000100:00000001:1.0:1789466108.652112:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0001: PM QoS request active for cpu 3, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.652113:0:117762:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789466108.652120:0:117762:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.652123:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.652125:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.652129:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466108.652130:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.652132:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466108.652133:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466108.652134:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.652136:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1789466108.652221:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.652227:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466108.652229:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.652233:0:134719:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1789466108.652235:0:134719:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1789466108.652237:0:134719:0:(import.c:65:import_set_state_nolock()) ffff8966c6120000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1789466108.652240:0:134719:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1789466108.652244:0:134719:0:(import.c:545:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 18336 00000100:00001000:0.0:1789466108.652248:0:134719:0:(import.c:2196:obd_at_measure()) add 20 to ffff8966c6120418 time=142 v=15 (15 1 0 0) 00000100:00001000:0.0:1789466108.652251:0:134719:0:(import.c:2262:obd_at_measure()) AT ffff8966c6120418 change: old=15 new=20 delta=5 (val=20) hist 20 1 0 0 00000100:00080000:0.0:1789466108.652254:0:134719:0:(import.c:630:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00000040:0.0:1789466108.652257:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.652259:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.652261:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.652263:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000001:0.0:1789466108.652268:0:134719:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466108.652270:0:134719:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b7944cf 00000020:00000001:0.0:1789466108.652272:0:134719:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466108.652276:0:134719:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8966c6127800 refcount=8 00000020:00000001:0.0:1789466108.652278:0:134719:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613673235609600 : -130400473942016 : ffff8966c6127800) 00000020:00000001:0.0:1789466108.652280:0:134719:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613673235609600 : -130400473942016 : ffff8966c6127800) 00000100:00000040:0.0:1789466108.652283:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.652285:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.652286:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.652288:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000040:0.0:1789466108.652299:0:134719:0:(genops.c:879:class_export_put()) PUTting export ffff8966c6127800 : new refcount 7 00000100:00080000:0.0:1789466108.652302:0:134719:0:(import.c:666:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8966c6120000 using connection 192.168.203.142@tcp/0@lo 00000100:00000001:0.0:1789466108.652304:0:134719:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466108.652307:0:134719:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1789466108.652313:0:134719:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789466108.652315:0:134719:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.652318:0:134719:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1789466108.652320:0:134719:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1789466108.652324:0:134719:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8967cb942d80. 00000020:00000040:0.0:1789466108.652328:0:134719:0:(genops.c:1209:class_import_get()) import ffff8966c6120000 refcount=3 obd=lustre-MDT0000-lwp-OST0000 02000000:00000001:0.0:1789466108.652331:0:134719:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1789466108.652333:0:134719:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.652336:0:134719:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1789466108.652338:0:134719:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1789466108.652341:0:134719:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1789466108.652347:0:134719:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8967ee8ff400. 00000100:00000001:0.0:1789466108.652355:0:134719:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466108.652360:0:134719:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8967cb942d80 x1876386404769152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1789466108.652368:0:134719:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1789466108.652371:0:134719:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466108.652375:0:134719:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bc0c79] to pc [ptlrpcd_rcv+-1] req@ffff8967cb942d80 x1876386404769152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789466108.652396:0:134719:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1789466108.652399:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466108.652402:0:134719:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1789466108.652404:0:134719:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1789466108.652405:0:134719:0:(import.c:65:import_set_state_nolock()) ffff8966c6123800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1789466108.652408:0:134719:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1789466108.652410:0:134719:0:(import.c:545:import_select_connection()) lustre-MDT0001-lwp-OST0000: connect to NID 0@lo last attempt 18336 00000100:00001000:0.0:1789466108.652413:0:134719:0:(import.c:2196:obd_at_measure()) add 20 to ffff8966c6123c18 time=25 v=15 (15 10 1 1) 00000100:00001000:0.0:1789466108.652416:0:134719:0:(import.c:2262:obd_at_measure()) AT ffff8966c6123c18 change: old=15 new=20 delta=5 (val=20) hist 20 10 1 1 00000100:00080000:0.0:1789466108.652425:0:134719:0:(import.c:630:import_select_connection()) lustre-MDT0001-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00000040:0.0:1789466108.652427:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.652430:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.652431:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.652433:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000001:0.0:1789466108.652437:0:134719:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466108.652438:0:134719:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b7944e4 00000020:00000001:0.0:1789466108.652440:0:134719:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466108.652442:0:134719:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967d0b2f000 refcount=5 00000020:00000001:0.0:1789466108.652445:0:134719:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677708865536 : -130396000686080 : ffff8967d0b2f000) 00000020:00000001:0.0:1789466108.652447:0:134719:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677708865536 : -130396000686080 : ffff8967d0b2f000) 00000100:00000040:0.0:1789466108.652449:0:134719:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:0.0:1789466108.652451:0:134719:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466108.652453:0:134719:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:0.0:1789466108.652455:0:134719:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000040:0.0:1789466108.652456:0:134719:0:(genops.c:879:class_export_put()) PUTting export ffff8967d0b2f000 : new refcount 4 00000100:00080000:0.0:1789466108.652459:0:134719:0:(import.c:666:import_select_connection()) lustre-MDT0001-lwp-OST0000: import ffff8966c6123800 using connection 192.168.203.142@tcp/0@lo 00000100:00000001:0.0:1789466108.652461:0:134719:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466108.652462:0:134719:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1789466108.652465:0:134719:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789466108.652467:0:134719:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.652469:0:134719:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1789466108.652470:0:134719:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1789466108.652474:0:134719:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8967cb940380. 00000020:00000040:0.0:1789466108.652476:0:134719:0:(genops.c:1209:class_import_get()) import ffff8966c6123800 refcount=3 obd=lustre-MDT0001-lwp-OST0000 02000000:00000001:0.0:1789466108.652478:0:134719:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1789466108.652479:0:134719:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466108.652481:0:134719:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1789466108.652483:0:134719:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1789466108.652485:0:134719:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1789466108.652488:0:134719:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8967ee8fe000. 00000100:00000001:0.0:1789466108.652491:0:134719:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466108.652503:0:134719:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8967cb940380 x1876386404769280/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1789466108.652522:0:134719:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1789466108.652524:0:134719:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466108.652527:0:134719:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034057c82] to pc [ptlrpcd_rcv+-1] req@ffff8967cb940380 x1876386404769280/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789466108.652534:0:134719:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466108.652539:0:134719:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18361) 00000100:00000001:1.0:1789466108.656630:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466108.656646:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466108.656650:0:117762:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789466108.656668:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8967cb942d80 x1876386404769152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.656685:0:117762:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789466108.656688:0:117762:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.656694:0:117762:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789466108.656698:0:117762:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466108.656702:0:117762:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466108.656706:0:117762:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8967cb942d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:117762:1876386404769152:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.656711:0:117762:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789466108.656717:0:117762:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789466108.656720:0:117762:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.656723:0:117762:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789466108.656736:0:117762:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8967fe7f1400. 02000000:00000001:1.0:1789466108.656739:0:117762:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1789466108.656750:0:117762:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8967f4dbf0b0. 00000400:00000010:1.0:1789466108.656759:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c63513b8. 00000100:00000200:1.0:1789466108.656767:0:117762:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1876386404769152, portal 10 00000100:00000001:1.0:1789466108.656770:0:117762:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789466108.656772:0:117762:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613677622963584 : -130396086588032 : ffff8967cb942d80) 00000100:00000040:1.0:1789466108.656781:0:117762:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8967cb942d80 x1876386404769152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.656790:0:117762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466108.656793:0:117762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1789466108.656816:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351aa0. 00000100:00000200:1.0:1789466108.656821:0:117762:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1876386404769152, offset 0 00000400:00000200:1.0:1789466108.656830:0:117762:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1789466108.656848:0:117762:0:(lib-move.c:4645:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1789466108.656863:0:117762:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 520 into portal 12 MB=0x6aa90411e1d80 00000400:00000100:1.0:1789466108.656870:0:117762:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 12 match 1876386404769152 offset 0 length 520: 4 00000400:00000200:1.0:1789466108.656876:0:117762:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8966e263dfe0 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1789466108.656879:0:117762:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:1.0:1789466108.656895:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351aa0 00000400:00000010:1.0:1789466108.656898:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351aa0. 00000100:00000001:1.0:1789466108.656903:0:117762:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1789466108.656908:0:117762:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8967cb942d80 x1876386404769152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.656916:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.656920:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8967cb942d80 x1876386404769152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.656926:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.656928:0:117762:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1789466108.656933:0:117762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466108.656938:0:117762:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: using a timeout of 18622 usecs (19 jiffies) 00000100:00000001:1.0:1789466108.656943:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS new deadline estimation for cpu 0 is 4313024092 00000100:00000001:1.0:1789466108.656946:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS request active for cpu 0, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.656949:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS new deadline estimation for cpu 1 is 4313024092 00000100:00000001:1.0:1789466108.656951:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS request active for cpu 1, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.656953:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS new deadline estimation for cpu 2 is 4313024092 00000100:00000001:1.0:1789466108.656955:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS request active for cpu 2, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.656957:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS new deadline estimation for cpu 3 is 4313024092 00000100:00000001:1.0:1789466108.656959:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-OST0000: PM QoS request active for cpu 3, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.656961:0:117762:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789466108.656964:0:117762:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.656974:0:117762:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789466108.656978:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8967cb940380 x1876386404769280/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.656984:0:117762:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789466108.656986:0:117762:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.656988:0:117762:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789466108.656990:0:117762:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466108.656992:0:117762:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466108.656994:0:117762:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8967cb940380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-lwp-OST0000_UUID:117762:1876386404769280:0@lo:38:kworker.0 00000100:00000001:1.0:1789466108.656997:0:117762:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789466108.657000:0:117762:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789466108.657001:0:117762:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466108.657003:0:117762:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789466108.657006:0:117762:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8967fe7f0000. 02000000:00000001:1.0:1789466108.657008:0:117762:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1789466108.657011:0:117762:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8967f4dbfe70. 00000400:00000010:1.0:1789466108.657014:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351aa0. 00000100:00000200:1.0:1789466108.657017:0:117762:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1876386404769280, portal 10 00000100:00000001:1.0:1789466108.657019:0:117762:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789466108.657020:0:117762:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613677622952832 : -130396086598784 : ffff8967cb940380) 00000100:00000040:1.0:1789466108.657025:0:117762:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8967cb940380 x1876386404769280/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.657031:0:117762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466108.657033:0:117762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1789466108.657035:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351550. 00000100:00000200:1.0:1789466108.657073:0:117762:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1876386404769280, offset 0 00000400:00000200:1.0:1789466108.657078:0:117762:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1789466108.657083:0:117762:0:(lib-move.c:4645:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1789466108.657089:0:117762:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 520 into portal 12 MB=0x6aa90411e1e00 00000400:00000100:1.0:1789466108.657092:0:117762:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 12 match 1876386404769280 offset 0 length 520: 4 00000400:00000200:1.0:1789466108.657095:0:117762:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8966e263dfe0 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1789466108.657097:0:117762:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:1.0:1789466108.657101:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351550 00000400:00000010:1.0:1789466108.657109:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351550. 00000100:00000001:1.0:1789466108.657112:0:117762:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1789466108.657115:0:117762:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8967cb940380 x1876386404769280/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.657121:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466108.657124:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8967cb940380 x1876386404769280/t0(0) o38->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 1789466133 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466108.657129:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.657130:0:117762:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1789466108.657134:0:117762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466108.657137:0:117762:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: using a timeout of 18658 usecs (19 jiffies) 00000100:00000001:1.0:1789466108.657139:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS new deadline estimation for cpu 0 is 4313024093 00000100:00000001:1.0:1789466108.657142:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS request active for cpu 0, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.657143:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS new deadline estimation for cpu 1 is 4313024093 00000100:00000001:1.0:1789466108.657145:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS request active for cpu 1, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.657147:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS new deadline estimation for cpu 2 is 4313024093 00000100:00000001:1.0:1789466108.657149:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS request active for cpu 2, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.657150:0:117762:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS new deadline estimation for cpu 3 is 4313024093 00000100:00000001:1.0:1789466108.657152:0:117762:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-lwp-OST0000: PM QoS request active for cpu 3, keep current deadline 4313024114 00000100:00000001:1.0:1789466108.657154:0:117762:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789466108.657156:0:117762:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.657161:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.657164:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.657174:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466108.657176:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466108.657179:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466108.657180:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466108.657182:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466108.657184:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466108.903435:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466108.903448:0:222999:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789466108.903451:0:222999:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789466108.903456:0:222999:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466108.903462:0:222999:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789466108.903465:0:222999:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466108.903468:0:222999:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466108.903470:0:222999:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466108.903560:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466108.903566:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466108.903595:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:2.0:1789466109.063817:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466109.063846:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466109.063850:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466109.063852:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466109.063867:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466109.063894:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94a00 00000400:00000200:2.0:1789466109.063905:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 5600 00000800:00000001:2.0:1789466109.063914:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466109.063918:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466109.063922:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466109.063948:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466109.063958:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466109.063961:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466109.063968:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189e300. 00000100:00000040:2.0:1789466109.063973:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff89680189e300 x1876386402355712 msgsize 224 00000100:00100000:2.0:1789466109.063977:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466109.064010:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466109.064024:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466109.064028:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466109.064030:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466109.064032:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466109.064082:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466109.064091:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94980 00000400:00000200:2.0:1789466109.064095:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 5824 00000800:00000001:2.0:1789466109.064100:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1789466109.064109:0:186366:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:2.0:1789466109.064112:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1789466109.064114:0:186366:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402355712 00000400:00000200:2.0:1789466109.064116:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:0.0:1789466109.064119:0:186366:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:2.0:1789466109.064122:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:0.0:1789466109.064124:0:186366:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789466109.064127:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466109.064130:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466109.064133:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189ce00. 00000100:00000040:2.0:1789466109.064135:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff89680189ce00 x1876386402355584 msgsize 224 00000100:00000001:0.0:1789466109.064137:0:186366:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789466109.064142:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 02000000:00000001:0.0:1789466109.064147:0:186366:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466109.064154:0:186366:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402355712 00000020:00000001:0.0:1789466109.064158:0:186366:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466109.064160:0:186366:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b8705ac 00000100:00000001:2.0:1789466109.064162:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466109.064168:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466109.064172:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1789466109.064256:0:186366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466109.064262:0:186366:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5cc5000 refcount=7 00000020:00000001:0.0:1789466109.064266:0:186366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000020:00000001:0.0:1789466109.064268:0:186366:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000100:00000001:0.0:1789466109.064275:0:186366:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466109.064281:0:186366:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e180. 00000020:00000010:0.0:1789466109.064286:0:186366:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ee05e880. 00000020:00000040:0.0:1789466109.064294:0:186366:0:(obd_config.c:995:class_incref()) incref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:0.0:1789466109.064314:0:186366:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:0.0:1789466109.064329:0:186366:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967d2e7d600. 00000020:00000010:0.0:1789466109.064337:0:186366:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3528d80. 00000020:00000010:0.0:1789466109.064343:0:186366:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967e94d8d48. 00000100:00000001:3.0:1789466109.064356:0:201486:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:0.0:1789466109.064359:0:186366:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00100000:3.0:1789466109.064365:0:201486:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402355584 00000100:00000001:0.0:1789466109.064366:0:186366:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1789466109.064369:0:186366:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1789466109.064376:0:201486:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1789466109.064376:0:186366:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466109.064381:0:201486:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789466109.064385:0:201486:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466109.064389:0:186366:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:3.0:1789466109.064392:0:201486:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466109.064395:0:186366:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00100000:3.0:1789466109.064399:0:201486:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402355584 00000100:00000001:0.0:1789466109.064400:0:186366:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466109.064408:0:201486:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1789466109.064412:0:201486:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b86f3e6 00000020:00000001:3.0:1789466109.064415:0:201486:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1789466109.064419:0:201486:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967c3b5d800 refcount=7 00000020:00000001:3.0:1789466109.064422:0:201486:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000020:00000001:3.0:1789466109.064425:0:201486:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000100:00000001:0.0:1789466109.064425:0:186366:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466109.064438:0:201486:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466109.064448:0:201486:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657c2c0. 00000100:00000001:0.0:1789466109.064449:0:186366:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1789466109.064456:0:186366:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:3.0:1789466109.064459:0:201486:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ee05e940. 00000100:00100000:0.0:1789466109.064468:0:186366:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3505 00000020:00000040:3.0:1789466109.064473:0:201486:0:(obd_config.c:995:class_incref()) incref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000040:0.0:1789466109.064475:0:186366:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967f5cc5000 : new rpc_count 1 00000100:00000001:0.0:1789466109.064481:0:186366:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528258816 : -130395181292800 : ffff89680189e300) 00000100:00000001:3.0:1789466109.064484:0:201486:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:3.0:1789466109.064492:0:201486:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967f5dd7400. 00000020:00000010:3.0:1789466109.064499:0:201486:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3614f00. 00000020:00000010:3.0:1789466109.064513:0:201486:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8966c68cfc80. 00000100:00000001:3.0:1789466109.064522:0:201486:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1789466109.064524:0:201486:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1789466109.064527:0:201486:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466109.064534:0:201486:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1789466109.064536:0:201486:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1789466109.064538:0:201486:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466109.064554:0:201486:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466109.064566:0:201486:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1789466109.064568:0:201486:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1789466109.064575:0:201486:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3506 00000100:00000040:3.0:1789466109.064578:0:201486:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967c3b5d800 : new rpc_count 1 00000100:00000001:3.0:1789466109.064580:0:201486:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528253440 : -130395181298176 : ffff89680189ce00) 00000100:00000040:3.0:1789466109.064588:0:201486:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189ce00 x1876386402355584/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:605/0 lens 224/0 e 0 to 0 dl 1789466120 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466109.064613:0:201486:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466109.064615:0:201486:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657cc40. 00000100:00000001:3.0:1789466109.064618:0:201486:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1789466109.064620:0:201486:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967c657cc40. 00000100:00100000:3.0:1789466109.064625:0:201486:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253749:x1876386402355584:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:3.0:1789466109.064629:0:201486:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402355584 00000020:00000001:3.0:1789466109.064633:0:201486:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466109.064636:0:201486:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1789466109.064638:0:201486:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466109.064640:0:201486:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1789466109.064642:0:201486:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:3.0:1789466109.064645:0:201486:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1789466109.064648:0:201486:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1789466109.064650:0:201486:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1789466109.064652:0:201486:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1789466109.064656:0:201486:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466109.064657:0:201486:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1789466109.064662:0:201486:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1789466109.064664:0:201486:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1789466109.064668:0:201486:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c7459000. 02000000:00000001:3.0:1789466109.064670:0:201486:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466109.064672:0:201486:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466109.064674:0:201486:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1789466109.064676:0:201486:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 17179870928, transno 0, xid 1876386402355584 00010000:00000001:3.0:1789466109.064679:0:201486:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:3.0:1789466109.064684:0:201486:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189ce00 x1876386402355584/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:605/0 lens 224/224 e 0 to 0 dl 1789466120 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789466109.064691:0:201486:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1789466109.064693:0:201486:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789466109.064697:0:201486:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=18 v=5 (1 1 1 1) 00000100:00000001:3.0:1789466109.064701:0:201486:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466109.064708:0:201486:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:3.0:1789466109.064710:0:201486:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:3.0:1789466109.064712:0:201486:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1789466109.064714:0:201486:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466109.064718:0:201486:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789466109.064720:0:201486:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:3.0:1789466109.064725:0:201486:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed93ab28. 00000100:00000200:3.0:1789466109.064731:0:201486:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402355584, offset 224 00000400:00000200:3.0:1789466109.064737:0:201486:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:3.0:1789466109.064750:0:201486:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:3.0:1789466109.064757:0:201486:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93557:93557:256:4294967295] 192.168.203.42@tcp LPNI seq info [93553:93553:8:4294967295] 00000400:00000200:3.0:1789466109.064768:0:201486:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:3.0:1789466109.064773:0:201486:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:3.0:1789466109.064777:0:201486:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966db009600. 00000800:00000200:3.0:1789466109.064783:0:201486:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:3.0:1789466109.064791:0:201486:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:3.0:1789466109.064795:0:201486:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966db009600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789466109.064807:0:201486:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789466109.064811:0:201486:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:3.0:1789466109.064813:0:201486:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:3.0:1789466109.064815:0:201486:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466109.064817:0:201486:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1789466109.064821:0:201486:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189ce00 x1876386402355584/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:605/0 lens 224/224 e 0 to 0 dl 1789466120 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789466109.064829:0:201486:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253749:x1876386402355584:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 216us (692us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466109.064853:0:201486:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3506 00000100:00000040:3.0:1789466109.064856:0:201486:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967c3b5d800 : new rpc_count 0 00000100:00000001:3.0:1789466109.064859:0:201486:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1789466109.064866:0:201486:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1789466109.064871:0:201486:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3614f00. 00000020:00000010:3.0:1789466109.064875:0:201486:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8966c68cfc80. 00000020:00000010:3.0:1789466109.064878:0:201486:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967f5dd7400. 00000020:00000040:3.0:1789466109.064882:0:201486:0:(genops.c:879:class_export_put()) PUTting export ffff8967c3b5d800 : new refcount 6 00000100:00000001:3.0:1789466109.064885:0:201486:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1789466109.064979:0:117757:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789466109.064982:0:117757:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966db009600. 00000400:00000200:3.0:1789466109.064987:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466109.064992:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:3.0:1789466109.064997:0:117757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed93ab28 00000400:00000010:3.0:1789466109.065000:0:117757:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed93ab28. 00000100:00000001:3.0:1789466109.065004:0:117757:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:3.0:1789466109.065006:0:117757:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00080000:2.0:1789466109.065160:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000020:00000040:2.0:1789466109.065173:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00080000:2.0:1789466109.065177:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000020:00000040:2.0:1789466109.065180:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000040:0.0:1789466109.065784:0:186366:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189e300 x1876386402355712/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:605/0 lens 224/0 e 0 to 0 dl 1789466120 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789466109.065802:0:186366:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466109.065810:0:186366:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e880. 00000100:00000001:0.0:1789466109.065813:0:186366:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1789466109.065816:0:186366:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967ee05e880. 00000100:00100000:0.0:1789466109.065824:0:186366:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253750:x1876386402355712:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:0.0:1789466109.065827:0:186366:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402355712 00000020:00000001:0.0:1789466109.065832:0:186366:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1789466109.065836:0:186366:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1789466109.065839:0:186366:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466109.065841:0:186366:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1789466109.065844:0:186366:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:0.0:1789466109.065848:0:186366:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1789466109.065851:0:186366:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1789466109.065853:0:186366:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1789466109.065866:0:186366:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1789466109.065870:0:186366:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466109.065872:0:186366:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1789466109.065878:0:186366:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1789466109.065881:0:186366:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1789466109.065887:0:186366:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967ee8fd000. 02000000:00000001:0.0:1789466109.065890:0:186366:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466109.065894:0:186366:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466109.065896:0:186366:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1789466109.065900:0:186366:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 12884913986, transno 0, xid 1876386402355712 00010000:00000001:0.0:1789466109.065903:0:186366:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:0.0:1789466109.065909:0:186366:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189e300 x1876386402355712/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:605/0 lens 224/224 e 0 to 0 dl 1789466120 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1789466109.065918:0:186366:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1789466109.065921:0:186366:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789466109.065925:0:186366:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=18 v=5 (1 1 1 1) 00000100:00000001:0.0:1789466109.065930:0:186366:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1789466109.065933:0:186366:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:0.0:1789466109.065936:0:186366:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:0.0:1789466109.065939:0:186366:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1789466109.065941:0:186366:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466109.065945:0:186366:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789466109.065947:0:186366:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:0.0:1789466109.065953:0:186366:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967f7b93440. 00000100:00000200:0.0:1789466109.065960:0:186366:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402355712, offset 224 00000400:00000200:0.0:1789466109.065967:0:186366:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:0.0:1789466109.065980:0:186366:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:0.0:1789466109.065987:0:186366:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93558:93558:256:4294967295] 192.168.203.42@tcp LPNI seq info [93554:93554:8:4294967295] 00000400:00000200:0.0:1789466109.065999:0:186366:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:0.0:1789466109.066099:0:186366:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:0.0:1789466109.066105:0:186366:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8967c9f9e700. 00000800:00000200:0.0:1789466109.066113:0:186366:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:0.0:1789466109.066128:0:186366:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:0.0:1789466109.066287:0:186366:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8967c9f9e700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789466109.066307:0:186366:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789466109.066311:0:186366:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:0.0:1789466109.066314:0:186366:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:0.0:1789466109.066316:0:186366:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466109.066318:0:186366:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1789466109.066323:0:186366:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189e300 x1876386402355712/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:605/0 lens 224/224 e 0 to 0 dl 1789466120 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789466109.066334:0:186366:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253750:x1876386402355712:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 533us (2358us total) trans 0 rc 0/0 00000100:00100000:0.0:1789466109.066344:0:186366:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3505 00000100:00000040:0.0:1789466109.066348:0:186366:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967f5cc5000 : new rpc_count 0 00000100:00000001:0.0:1789466109.066351:0:186366:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1789466109.066352:0:186366:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1789466109.066359:0:186366:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3528d80. 00000020:00000010:0.0:1789466109.066364:0:186366:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967e94d8d48. 00000020:00000010:0.0:1789466109.066369:0:186366:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967d2e7d600. 00000020:00000040:0.0:1789466109.066375:0:186366:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5cc5000 : new refcount 6 00000100:00000001:0.0:1789466109.066379:0:186366:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1789466109.066822:0:117758:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789466109.066827:0:117758:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8967c9f9e700. 00000400:00000200:2.0:1789466109.066834:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466109.066848:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:2.0:1789466109.066856:0:117758:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967f7b93440 00000400:00000010:2.0:1789466109.066859:0:117758:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967f7b93440. 00000100:00000001:2.0:1789466109.066863:0:117758:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:2.0:1789466109.066865:0:117758:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00000040:3.0:1789466109.095188:0:218018:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37330 (cpu 3) has reached its deadline 4313024114, at 4313024512 00000100:00000010:3.0:1789466109.095203:0:218018:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc116c0. 00000100:00000040:1.0:1789466109.098781:0:222274:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37110 (cpu 1) has reached its deadline 4313024114, at 4313024515 00000100:00000010:1.0:1789466109.098802:0:222274:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc114e0. 00000100:00000040:0.0:1789466109.099071:0:222999:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37000 (cpu 0) has reached its deadline 4313024114, at 4313024516 00000100:00000010:0.0:1789466109.099077:0:222999:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc11f60. 00000100:00000040:2.0:1789466109.099160:0:222031:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37220 (cpu 2) has reached its deadline 4313024114, at 4313024516 00000100:00000010:2.0:1789466109.099168:0:222031:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc11de0. 00000100:00000001:1.0:1789466109.672017:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466109.672033:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466109.672034:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466109.672056:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466109.672060:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466109.672066:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466109.672069:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466109.927880:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466109.927890:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466109.927986:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466109.927990:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466109.927994:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466110.695392:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466110.695412:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466110.695415:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466110.695421:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466110.695424:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466110.695429:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466110.695432:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466110.951164:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466110.951174:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466110.951394:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789466110.951399:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789466110.951403:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466111.719188:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466111.719217:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466111.719220:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466111.719227:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466111.719230:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466111.719237:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466111.719240:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466111.976775:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466111.976785:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466111.976823:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466111.976849:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466111.976853:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466112.743677:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466112.743697:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466112.743700:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466112.743706:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466112.743709:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466112.743716:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466112.743719:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466113.000814:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466113.000824:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466113.000938:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466113.000944:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466113.000948:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789466113.704612:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466113.704627:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466113.704630:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466113.704634:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466113.704637:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466113.704640:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466113.704643:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466113.704647:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466113.704650:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466113.704679:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1789466113.704687:0:223621:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18366) 00000100:00000001:1.0:1789466113.767246:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466113.767272:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466113.767275:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466113.767282:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466113.767285:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466113.767292:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466113.767295:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466114.023238:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466114.023248:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466114.023285:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466114.023288:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466114.023291:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:2.0:1789466114.180379:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180408:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.180414:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180417:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466114.180433:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466114.180458:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94b00 00000400:00000200:2.0:1789466114.180481:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 6048 00000800:00000001:2.0:1789466114.180491:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180496:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466114.180499:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466114.180504:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466114.180514:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466114.180516:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466114.180525:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189df80. 00000100:00000040:2.0:1789466114.180539:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff89680189df80 x1876386402355968 msgsize 224 00000100:00100000:2.0:1789466114.180544:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466114.180567:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466114.180580:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180585:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.180587:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180589:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466114.180593:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466114.180599:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94a80 00000400:00000200:2.0:1789466114.180604:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 6272 00000800:00000001:2.0:1789466114.180608:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180618:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466114.180637:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466114.180641:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466114.180644:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466114.180646:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466114.180649:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189f100. 00000100:00000040:2.0:1789466114.180652:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff89680189f100 x1876386402355840 msgsize 224 00000100:00100000:2.0:1789466114.180656:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466114.180674:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466114.180679:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.180682:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466114.180915:0:187060:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1789466114.180919:0:187060:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402355968 02000000:00000001:0.0:1789466114.180923:0:187060:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1789466114.180926:0:187060:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789466114.180928:0:187060:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466114.180935:0:187060:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466114.180940:0:187060:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402355968 00000020:00000001:0.0:1789466114.180944:0:187060:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466114.180946:0:187060:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b8705ac 00000020:00000001:0.0:1789466114.180950:0:187060:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466114.180955:0:187060:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5cc5000 refcount=7 00000020:00000001:0.0:1789466114.180958:0:187060:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000020:00000001:0.0:1789466114.180961:0:187060:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000100:00000001:0.0:1789466114.180967:0:187060:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466114.180972:0:187060:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e880. 00000020:00000010:0.0:1789466114.180978:0:187060:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ee05e180. 00000020:00000040:0.0:1789466114.180987:0:187060:0:(obd_config.c:995:class_incref()) incref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:0.0:1789466114.181007:0:187060:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:0.0:1789466114.181014:0:187060:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967d2e7d600. 00000020:00000010:0.0:1789466114.181021:0:187060:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3528d80. 00000020:00000010:0.0:1789466114.181028:0:187060:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967e94d8d48. 00000100:00000040:0.0:1789466114.181078:0:187060:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00000001:0.0:1789466114.181083:0:187060:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1789466114.181084:0:187060:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1789466114.181087:0:187060:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466114.181096:0:187060:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1789466114.181107:0:187060:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1789466114.181109:0:187060:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789466114.181179:0:187060:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466114.181192:0:187060:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1789466114.181194:0:187060:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402355840 02000000:00000001:0.0:1789466114.181197:0:187060:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1789466114.181198:0:187060:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789466114.181200:0:187060:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789466114.181202:0:187060:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789466114.181205:0:187060:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402355840 00000020:00000001:0.0:1789466114.181206:0:187060:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1789466114.181208:0:187060:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b86f3e6 00000020:00000001:0.0:1789466114.181209:0:187060:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1789466114.181212:0:187060:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967c3b5d800 refcount=7 00000020:00000001:0.0:1789466114.181214:0:187060:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000020:00000001:0.0:1789466114.181216:0:187060:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000100:00000001:0.0:1789466114.181219:0:187060:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1789466114.181222:0:187060:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967ee05e180. 00000020:00000010:0.0:1789466114.181225:0:187060:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967c657c2c0. 00000020:00000040:0.0:1789466114.181229:0:187060:0:(obd_config.c:995:class_incref()) incref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000001:0.0:1789466114.181231:0:187060:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:0.0:1789466114.181235:0:187060:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967d2e7d000. 00000020:00000010:0.0:1789466114.181237:0:187060:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3528780. 00000020:00000010:0.0:1789466114.181240:0:187060:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967e94d8ed8. 00000100:00000001:0.0:1789466114.181244:0:187060:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1789466114.181246:0:187060:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1789466114.181247:0:187060:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789466114.181250:0:187060:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1789466114.181252:0:187060:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1789466114.181253:0:187060:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1789466114.181706:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000020:00000040:2.0:1789466114.181712:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00080000:2.0:1789466114.181716:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000020:00000040:2.0:1789466114.181718:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:3.0:1789466114.182293:0:121893:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1789466114.182297:0:121893:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1789466114.182309:0:121893:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3507 00000100:00000040:3.0:1789466114.182314:0:121893:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967f5cc5000 : new rpc_count 1 00000100:00000001:3.0:1789466114.182317:0:121893:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528257920 : -130395181293696 : ffff89680189df80) 00000100:00000040:3.0:1789466114.182329:0:121893:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189df80 x1876386402355968/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:610/0 lens 224/0 e 0 to 0 dl 1789466125 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466114.182342:0:121893:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466114.182351:0:121893:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657cc40. 00000100:00000001:3.0:1789466114.182354:0:121893:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1789466114.182357:0:121893:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967c657cc40. 00000100:00100000:3.0:1789466114.182364:0:121893:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402355968:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:3.0:1789466114.182368:0:121893:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402355968 00000020:00000001:3.0:1789466114.182372:0:121893:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466114.182377:0:121893:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1789466114.182380:0:121893:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.182382:0:121893:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1789466114.182386:0:121893:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:3.0:1789466114.182389:0:121893:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1789466114.182393:0:121893:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1789466114.182395:0:121893:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1789466114.182398:0:121893:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1789466114.182402:0:121893:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.182404:0:121893:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1789466114.182410:0:121893:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1789466114.182413:0:121893:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1789466114.182420:0:121893:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c7459800. 02000000:00000001:3.0:1789466114.182423:0:121893:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466114.182426:0:121893:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.182442:0:121893:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1789466114.182446:0:121893:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 12884913986, transno 0, xid 1876386402355968 00010000:00000001:3.0:1789466114.182450:0:121893:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:3.0:1789466114.182456:0:121893:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189df80 x1876386402355968/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:610/0 lens 224/224 e 0 to 0 dl 1789466125 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789466114.182537:0:121893:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1789466114.182541:0:121893:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789466114.182547:0:121893:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=23 v=5 (1 1 1 1) 00000100:00000001:3.0:1789466114.182552:0:121893:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466114.182555:0:121893:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:3.0:1789466114.182559:0:121893:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:3.0:1789466114.182562:0:121893:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1789466114.182565:0:121893:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466114.182569:0:121893:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789466114.182573:0:121893:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:3.0:1789466114.182580:0:121893:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed93ab28. 00000100:00000200:3.0:1789466114.182589:0:121893:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402355968, offset 224 00000400:00000200:3.0:1789466114.182598:0:121893:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:3.0:1789466114.182692:0:121893:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:3.0:1789466114.182702:0:121893:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93559:93559:256:4294967295] 192.168.203.42@tcp LPNI seq info [93555:93555:8:4294967295] 00000400:00000200:3.0:1789466114.182719:0:121893:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:3.0:1789466114.182727:0:121893:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:3.0:1789466114.182734:0:121893:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966db009600. 00000800:00000200:3.0:1789466114.182743:0:121893:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:3.0:1789466114.182754:0:121893:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:3.0:1789466114.182760:0:121893:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966db009600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789466114.182858:0:121893:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789466114.182863:0:121893:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:3.0:1789466114.182866:0:121893:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:3.0:1789466114.182868:0:121893:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.182870:0:121893:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1789466114.182882:0:121893:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189df80 x1876386402355968/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:610/0 lens 224/224 e 0 to 0 dl 1789466125 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1789466114.183028:0:117757:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789466114.183034:0:117757:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966db009600. 00000400:00000200:2.0:1789466114.183073:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466114.183079:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:2.0:1789466114.183086:0:117757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed93ab28 00000400:00000010:2.0:1789466114.183089:0:117757:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed93ab28. 00000100:00000001:2.0:1789466114.183094:0:117757:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:2.0:1789466114.183095:0:117757:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00100000:3.0:1789466114.183597:0:121893:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402355968:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 1252us (3053us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466114.183609:0:121893:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3507 00000100:00000040:3.0:1789466114.183721:0:121893:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967f5cc5000 : new rpc_count 0 00000100:00000001:3.0:1789466114.183725:0:121893:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1789466114.183728:0:121893:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1789466114.183737:0:121893:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3528d80. 00000020:00000010:3.0:1789466114.183742:0:121893:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967e94d8d48. 00000020:00000010:3.0:1789466114.183746:0:121893:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967d2e7d600. 00000020:00000040:3.0:1789466114.183753:0:121893:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5cc5000 : new refcount 6 00000100:00000001:3.0:1789466114.183756:0:121893:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466114.183849:0:121893:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1789466114.183851:0:121893:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1789466114.183857:0:121893:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3508 00000100:00000040:3.0:1789466114.183861:0:121893:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967c3b5d800 : new rpc_count 1 00000100:00000001:3.0:1789466114.183864:0:121893:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528262400 : -130395181289216 : ffff89680189f100) 00000100:00000040:3.0:1789466114.183871:0:121893:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189f100 x1876386402355840/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:610/0 lens 224/0 e 0 to 0 dl 1789466125 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466114.183881:0:121893:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466114.183981:0:121893:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657cc40. 00000100:00000001:3.0:1789466114.183986:0:121893:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1789466114.183990:0:121893:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967c657cc40. 00000100:00100000:3.0:1789466114.184287:0:121893:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402355840:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:3.0:1789466114.184292:0:121893:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402355840 00000020:00000001:3.0:1789466114.184296:0:121893:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466114.184301:0:121893:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1789466114.184304:0:121893:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.184321:0:121893:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1789466114.184323:0:121893:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:3.0:1789466114.184327:0:121893:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1789466114.184331:0:121893:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1789466114.184333:0:121893:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1789466114.184335:0:121893:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1789466114.184339:0:121893:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.184341:0:121893:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1789466114.184346:0:121893:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1789466114.184350:0:121893:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1789466114.184356:0:121893:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c745bc00. 02000000:00000001:3.0:1789466114.184359:0:121893:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466114.184362:0:121893:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.184365:0:121893:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1789466114.184368:0:121893:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 17179870928, transno 0, xid 1876386402355840 00010000:00000001:3.0:1789466114.184371:0:121893:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:3.0:1789466114.184377:0:121893:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189f100 x1876386402355840/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:610/0 lens 224/224 e 0 to 0 dl 1789466125 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789466114.184386:0:121893:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1789466114.184389:0:121893:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789466114.184393:0:121893:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=23 v=5 (1 1 1 1) 00000100:00000001:3.0:1789466114.184399:0:121893:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466114.184402:0:121893:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:3.0:1789466114.184404:0:121893:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:3.0:1789466114.184408:0:121893:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1789466114.184410:0:121893:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466114.184413:0:121893:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789466114.184416:0:121893:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:3.0:1789466114.184422:0:121893:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed93a880. 00000100:00000200:3.0:1789466114.184429:0:121893:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402355840, offset 224 00000400:00000200:3.0:1789466114.184436:0:121893:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:3.0:1789466114.184451:0:121893:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:3.0:1789466114.184541:0:121893:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93560:93560:256:4294967295] 192.168.203.42@tcp LPNI seq info [93556:93556:8:4294967295] 00000400:00000200:3.0:1789466114.184557:0:121893:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:3.0:1789466114.184571:0:121893:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:3.0:1789466114.184577:0:121893:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966db009100. 00000800:00000200:3.0:1789466114.184586:0:121893:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:3.0:1789466114.184593:0:121893:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:3.0:1789466114.184599:0:121893:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966db009100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789466114.184684:0:121893:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789466114.184688:0:121893:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:3.0:1789466114.184691:0:121893:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:3.0:1789466114.184693:0:121893:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466114.184695:0:121893:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1789466114.184700:0:121893:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189f100 x1876386402355840/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:610/0 lens 224/224 e 0 to 0 dl 1789466125 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789466114.184709:0:121893:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402355840:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 828us (4055us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466114.184716:0:121893:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3508 00000100:00000040:3.0:1789466114.184719:0:121893:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967c3b5d800 : new rpc_count 0 00000100:00000001:3.0:1789466114.184722:0:121893:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1789466114.184723:0:121893:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1789466114.184727:0:121893:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3528780. 00000020:00000010:3.0:1789466114.184731:0:121893:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967e94d8ed8. 00000020:00000010:3.0:1789466114.184734:0:121893:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967d2e7d000. 00000020:00000040:3.0:1789466114.184739:0:121893:0:(genops.c:879:class_export_put()) PUTting export ffff8967c3b5d800 : new refcount 6 00000100:00000001:3.0:1789466114.184742:0:121893:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1789466114.184793:0:117758:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789466114.184810:0:117758:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966db009100. 00000400:00000200:2.0:1789466114.184815:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466114.184821:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:2.0:1789466114.184827:0:117758:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed93a880 00000400:00000010:2.0:1789466114.184830:0:117758:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed93a880. 00000100:00000001:2.0:1789466114.184834:0:117758:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:2.0:1789466114.184836:0:117758:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00000001:0.0:1789466114.185203:0:187060:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.247546:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247572:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.247578:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247580:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466114.247595:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466114.247630:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 520 into portal 12 MB=0x6aa9040f94b80 00000400:00000100:2.0:1789466114.247660:0:117756:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-192.168.203.42@tcp portal 12 match 1876386402356096 offset 0 length 520: 4 00000400:00000200:2.0:1789466114.247668:0:117756:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8967e7afa420 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:2.0:1789466114.247671:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1789466114.247686:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247692:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.247695:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247697:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.247699:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247702:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466114.247706:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466114.247712:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 520 into portal 12 MB=0x6aa9040f94c00 00000400:00000100:2.0:1789466114.247716:0:117756:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-192.168.203.42@tcp portal 12 match 1876386402356224 offset 0 length 520: 4 00000400:00000200:2.0:1789466114.247719:0:117756:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8967e7afa420 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:2.0:1789466114.247721:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1789466114.247724:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247735:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466114.247737:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466114.247739:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466114.792856:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466114.792877:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466114.792903:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466114.792909:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466114.792912:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466114.792918:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466114.792921:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466115.049828:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466115.049837:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466115.050342:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789466115.050347:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789466115.050351:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466115.817249:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466115.817263:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466115.817265:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466115.817270:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466115.817272:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466115.817278:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466115.817280:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:0.0:1789466115.863625:0:1:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3528600. 00080000:00000010:0.0:1789466115.863637:0:1:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3528600. 00000020:00000001:0.0:1789466115.863648:0:1:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:0.0:1789466115.863657:0:1:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3528600. 00080000:00000010:0.0:1789466115.863663:0:1:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3528600. 00000020:00000001:0.0:1789466115.863667:0:1:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466116.072139:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466116.072146:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466116.072172:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466116.072175:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466116.072178:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466116.839237:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466116.839253:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466116.839255:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466116.839259:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466116.839261:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466116.839266:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466116.839268:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466117.095392:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466117.095408:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466117.096207:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466117.096212:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466117.096216:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466117.863494:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466117.863507:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466117.863509:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466117.863513:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466117.863516:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466117.863521:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466117.863523:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466118.119486:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466118.119493:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466118.119519:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466118.119522:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466118.119525:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789466118.823173:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.823183:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.823186:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.823209:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.823211:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.823214:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.823217:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.823220:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.823223:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.823226:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1789466118.823232:0:223621:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18371) 00000100:00000001:1.0:1789466118.887162:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466118.887171:0:117762:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1789466118.887186:0:117762:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789466103/real 1789466103] req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1789466118.887203:0:117762:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789466118.887205:0:117762:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789466118.887209:0:117762:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8967f4dbfc08. 00000400:00000200:1.0:1789466118.887214:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351198 00000400:00000010:1.0:1789466118.887217:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351198. 00000100:00000001:1.0:1789466118.887238:0:117762:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789466118.887242:0:117762:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789466118.887249:0:117762:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.887256:0:117762:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789466118.887257:0:117762:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.887260:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466118.887262:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1789466118.887265:0:117762:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.887270:0:117762:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.887279:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466118.887281:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.887284:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466118.887286:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466118.887290:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789466118.887293:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.887299:0:117762:0:(client.c:2143:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1789466118.887302:0:117762:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.887303:0:117762:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789466118.887304:0:117762:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.887307:0:117762:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1789466118.887309:0:117762:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1789466118.887312:0:117762:0:(import.c:65:import_set_state_nolock()) ffff8967e7ba6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789466118.887341:0:117762:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00000001:1.0:1789466118.887343:0:117762:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1789466118.887347:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466118.887354:0:117762:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8967d3738a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:4861e2df-2452-4fd5-a0b1-355ea409d38c:117762:1876386404768768:0@lo:250:kworker.0 00000100:00000001:1.0:1789466118.887364:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466118.887367:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466118.887370:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8967d3738a80 x1876386404768768/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1789466118 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.887375:0:117762:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789466118.887378:0:117762:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789466118.887381:0:117762:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8967fe7f0c00. 02000000:00000001:1.0:1789466118.887386:0:117762:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789466118.887388:0:117762:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789466118.887390:0:117762:0:(genops.c:1222:class_import_put()) import ffff8967e7ba6800 refcount=3 obd=MGC192.168.203.142@tcp 00000020:00000001:1.0:1789466118.887392:0:117762:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789466118.887394:0:117762:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8967fe7f2000. 02000000:00000001:1.0:1789466118.887398:0:117762:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789466118.887399:0:117762:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789466118.887401:0:117762:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8967d3738a80. 00000100:00000001:1.0:1789466118.887404:0:117762:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789466118.887405:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.887407:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1789466118.887492:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.887497:0:223621:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1789466118.887500:0:223621:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1789466118.887502:0:223621:0:(import.c:65:import_set_state_nolock()) ffff8967e7ba6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1789466118.887505:0:223621:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1789466118.887508:0:223621:0:(import.c:545:import_select_connection()) MGC192.168.203.142@tcp: connect to NID 0@lo last attempt 18351 00000100:00001000:3.0:1789466118.887513:0:223621:0:(import.c:2196:obd_at_measure()) add 15 to ffff8967e7ba6c18 time=157 v=10 (10 1 0 0) 00000100:00001000:3.0:1789466118.887516:0:223621:0:(import.c:2262:obd_at_measure()) AT ffff8967e7ba6c18 change: old=10 new=15 delta=5 (val=15) hist 15 10 1 0 00000100:00080000:3.0:1789466118.887519:0:223621:0:(import.c:630:import_select_connection()) MGC192.168.203.142@tcp: tried all connections, increasing latency to 10s 00000100:00000040:3.0:1789466118.887521:0:223621:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:3.0:1789466118.887524:0:223621:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466118.887526:0:223621:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:3.0:1789466118.887528:0:223621:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000001:3.0:1789466118.887535:0:223621:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1789466118.887537:0:223621:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b794228 00000020:00000001:3.0:1789466118.887540:0:223621:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1789466118.887542:0:223621:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967e7ba1800 refcount=3 00000020:00000001:3.0:1789466118.887544:0:223621:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678095210496 : -130395614341120 : ffff8967e7ba1800) 00000020:00000001:3.0:1789466118.887554:0:223621:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678095210496 : -130395614341120 : ffff8967e7ba1800) 00000100:00000040:3.0:1789466118.887557:0:223621:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967f4d8bc00 refcount 14 to 0@lo 00000100:00000001:3.0:1789466118.887559:0:223621:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466118.887560:0:223621:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967f4d8bc00 refcount 15 to 0@lo 00000100:00000001:3.0:1789466118.887562:0:223621:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678315322368 : -130395394229248 : ffff8967f4d8bc00) 00000020:00000040:3.0:1789466118.887564:0:223621:0:(genops.c:879:class_export_put()) PUTting export ffff8967e7ba1800 : new refcount 2 00000100:00080000:3.0:1789466118.887567:0:223621:0:(import.c:666:import_select_connection()) MGC192.168.203.142@tcp: import ffff8967e7ba6800 using connection 192.168.203.142@tcp/0@lo 00000100:00000001:3.0:1789466118.887569:0:223621:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789466118.887571:0:223621:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1789466118.887574:0:223621:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789466118.887577:0:223621:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466118.887579:0:223621:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1789466118.887581:0:223621:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1789466118.887586:0:223621:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8966d3fe0e00. 00000020:00000040:3.0:1789466118.887589:0:223621:0:(genops.c:1209:class_import_get()) import ffff8967e7ba6800 refcount=4 obd=MGC192.168.203.142@tcp 02000000:00000001:3.0:1789466118.887592:0:223621:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1789466118.887594:0:223621:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466118.887596:0:223621:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789466118.887598:0:223621:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789466118.887602:0:223621:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789466118.887608:0:223621:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8967c745a800. 00000100:00000001:3.0:1789466118.887614:0:223621:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789466118.887620:0:223621:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8966d3fe0e00 x1876386404769408/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789466118.887628:0:223621:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789466118.887632:0:223621:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789466118.887635:0:223621:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b56ac6e] to pc [ptlrpcd_rcv+-1] req@ffff8966d3fe0e00 x1876386404769408/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466118.887654:0:223621:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1789466118.887658:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.887661:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.887670:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.887673:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.887675:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.887678:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789466118.887680:0:223621:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789466118.887683:0:223621:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1789466118.887687:0:223621:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18372) 00000100:00000001:1.0:1789466118.888146:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466118.888149:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466118.888151:0:117762:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789466118.888157:0:117762:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8966d3fe0e00 x1876386404769408/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.888163:0:117762:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789466118.888165:0:117762:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466118.888169:0:117762:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789466118.888171:0:117762:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789466118.888174:0:117762:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466118.888176:0:117762:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8966d3fe0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:4861e2df-2452-4fd5-a0b1-355ea409d38c:117762:1876386404769408:0@lo:250:kworker.0 00000100:00000001:1.0:1789466118.888180:0:117762:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789466118.888185:0:117762:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789466118.888187:0:117762:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466118.888189:0:117762:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789466118.888194:0:117762:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8967fe7f2000. 02000000:00000001:1.0:1789466118.888202:0:117762:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1789466118.888207:0:117762:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8967f4dbfc08. 00000400:00000010:1.0:1789466118.888212:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351198. 00000100:00000200:1.0:1789466118.888217:0:117762:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1876386404769408, portal 25 00000100:00000001:1.0:1789466118.888220:0:117762:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789466118.888221:0:117762:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613673469152768 : -130400240398848 : ffff8966d3fe0e00) 00000100:00000040:1.0:1789466118.888228:0:117762:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8966d3fe0e00 x1876386404769408/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 1789466138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.888234:0:117762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466118.888236:0:117762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1789466118.888239:0:117762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351550. 00000100:00000200:1.0:1789466118.888241:0:117762:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1876386404769408, offset 0 00000400:00000200:1.0:1789466118.888247:0:117762:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1789466118.888259:0:117762:0:(lib-move.c:4645:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1789466118.888269:0:117762:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 520 into portal 26 MB=0x6aa90411e1e80 00000400:00000100:1.0:1789466118.888275:0:117762:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1876386404769408 offset 0 length 520: 4 00000400:00000200:1.0:1789466118.888280:0:117762:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8966e263dfe0 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1789466118.888283:0:117762:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:1.0:1789466118.888293:0:117762:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351550 00000400:00000010:1.0:1789466118.888296:0:117762:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351550. 00000100:00000001:1.0:1789466118.888300:0:117762:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1789466118.888303:0:117762:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8966d3fe0e00 x1876386404769408/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 1789466138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.888311:0:117762:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789466118.888314:0:117762:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8966d3fe0e00 x1876386404769408/t0(0) o250->MGC192.168.203.142@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 1789466138 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466118.888320:0:117762:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466118.888321:0:117762:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1789466118.888325:0:117762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466118.888329:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 0 is 4313039475 00000100:00000010:1.0:1789466118.888334:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc114e0. 00000100:00000001:1.0:1789466118.888340:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1789466118.888355:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 1 is 4313039475 00000100:00000010:1.0:1789466118.888358:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc11600. 00000100:00000001:1.0:1789466118.888361:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1789466118.888364:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 2 is 4313039475 00000100:00000010:1.0:1789466118.888366:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc11cc0. 00000100:00000001:1.0:1789466118.888369:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1789466118.888372:0:117762:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS new deadline estimation for cpu 3 is 4313039475 00000100:00000010:1.0:1789466118.888374:0:117762:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8967dfc11180. 00000100:00000001:1.0:1789466118.888376:0:117762:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.142@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1789466118.888379:0:117762:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789466118.888382:0:117762:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466118.888385:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466118.888387:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.888393:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466118.888395:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466118.888397:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466118.888398:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466118.888399:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466118.888401:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466119.143207:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466119.143215:0:222999:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789466119.143218:0:222999:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789466119.143221:0:222999:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466119.143226:0:222999:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789466119.143228:0:222999:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466119.143230:0:222999:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466119.143233:0:222999:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466119.143327:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789466119.143330:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789466119.143333:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:2.0:1789466119.300857:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.300880:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466119.300885:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.300888:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466119.300902:0:117757:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466119.300924:0:117757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94c80 00000400:00000200:2.0:1789466119.300936:0:117757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 6496 00000800:00000001:2.0:1789466119.300945:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.300949:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466119.300954:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466119.300959:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466119.300968:0:117757:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466119.300972:0:117757:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466119.300981:0:117757:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189f480. 00000100:00000040:2.0:1789466119.301002:0:117757:0:(events.c:341:request_in_callback()) incoming req@ffff89680189f480 x1876386402356352 msgsize 224 00000100:00100000:2.0:1789466119.301008:0:117757:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466119.301088:0:117757:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466119.301100:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.301105:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466119.301107:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.301109:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466119.301114:0:117757:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466119.301120:0:117757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94d00 00000400:00000200:2.0:1789466119.301124:0:117757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 6720 00000800:00000001:2.0:1789466119.301128:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.301135:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466119.301138:0:117757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466119.301141:0:117757:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466119.301144:0:117757:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466119.301145:0:117757:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466119.301149:0:117757:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189f800. 00000100:00000040:2.0:1789466119.301151:0:117757:0:(events.c:341:request_in_callback()) incoming req@ffff89680189f800 x1876386402356480 msgsize 224 00000100:00100000:2.0:1789466119.301155:0:117757:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466119.301170:0:117757:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466119.301174:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.301178:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.301262:0:201493:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1789466119.301270:0:201493:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402356352 02000000:00000001:1.0:1789466119.301275:0:201493:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1789466119.301278:0:201493:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789466119.301280:0:201493:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466119.301286:0:201493:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466119.301290:0:201493:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402356352 00000020:00000001:1.0:1789466119.301294:0:201493:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1789466119.301296:0:201493:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b86f3e6 00000020:00000001:1.0:1789466119.301299:0:201493:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1789466119.301303:0:201493:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967c3b5d800 refcount=7 00000020:00000001:1.0:1789466119.301306:0:201493:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000020:00000001:1.0:1789466119.301326:0:201493:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000100:00000001:1.0:1789466119.301332:0:201493:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1789466119.301336:0:201493:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967f014f440. 00000020:00000010:1.0:1789466119.301341:0:201493:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ee05e180. 00000020:00000040:1.0:1789466119.301350:0:201493:0:(obd_config.c:995:class_incref()) incref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000001:1.0:1789466119.301375:0:201493:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:1.0:1789466119.301383:0:201493:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967c5c27a00. 00000020:00000010:1.0:1789466119.301389:0:201493:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d832ab80. 00000020:00000010:1.0:1789466119.301395:0:201493:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967f0ef3640. 00000100:00000040:1.0:1789466119.301406:0:201493:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00000001:1.0:1789466119.301409:0:201493:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1789466119.301410:0:201493:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1789466119.301413:0:201493:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.301421:0:201493:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1789466119.301435:0:201493:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1789466119.301436:0:201493:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466119.301452:0:201493:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.301464:0:201493:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1789466119.301466:0:201493:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402356480 02000000:00000001:1.0:1789466119.301468:0:201493:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1789466119.301469:0:201493:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789466119.301471:0:201493:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466119.301473:0:201493:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466119.301475:0:201493:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402356480 00000020:00000001:1.0:1789466119.301477:0:201493:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1789466119.301478:0:201493:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b8705ac 00000020:00000001:1.0:1789466119.301480:0:201493:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1789466119.301482:0:201493:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5cc5000 refcount=7 00000020:00000001:1.0:1789466119.301484:0:201493:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000020:00000001:1.0:1789466119.301486:0:201493:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000100:00000001:1.0:1789466119.301489:0:201493:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1789466119.301492:0:201493:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967f014f640. 00000020:00000010:1.0:1789466119.301495:0:201493:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967ee05e880. 00000020:00000040:1.0:1789466119.301499:0:201493:0:(obd_config.c:995:class_incref()) incref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:1.0:1789466119.301507:0:201493:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:1.0:1789466119.301511:0:201493:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967c5c26e00. 00000020:00000010:1.0:1789466119.301514:0:201493:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d832ac00. 00000020:00000010:1.0:1789466119.301517:0:201493:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967f0ef3898. 00000100:00000001:1.0:1789466119.301521:0:201493:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1789466119.301523:0:201493:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1789466119.301524:0:201493:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.301527:0:201493:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1789466119.301529:0:201493:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1789466119.301530:0:201493:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466119.301540:0:201493:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.301549:0:201493:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1789466119.301551:0:201493:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1789466119.301558:0:201493:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3509 00000100:00000040:1.0:1789466119.301562:0:201493:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967c3b5d800 : new rpc_count 1 00000100:00000001:1.0:1789466119.301565:0:201493:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528263296 : -130395181288320 : ffff89680189f480) 00000100:00000040:1.0:1789466119.301572:0:201493:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189f480 x1876386402356352/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:615/0 lens 224/0 e 0 to 0 dl 1789466130 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466119.301584:0:201493:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1789466119.301587:0:201493:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967f014fe00. 00000100:00000001:1.0:1789466119.301591:0:201493:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1789466119.301595:0:201493:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967f014fe00. 00000100:00100000:1.0:1789466119.301600:0:201493:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402356352:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:1.0:1789466119.301604:0:201493:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402356352 00000020:00000001:1.0:1789466119.301608:0:201493:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1789466119.301611:0:201493:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1789466119.301614:0:201493:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466119.301616:0:201493:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1789466119.301619:0:201493:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:1.0:1789466119.301622:0:201493:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1789466119.301625:0:201493:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1789466119.301627:0:201493:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1789466119.301637:0:201493:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1789466119.301640:0:201493:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466119.301643:0:201493:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1789466119.301649:0:201493:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1789466119.301651:0:201493:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1789466119.301658:0:201493:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967fe7f0c00. 02000000:00000001:1.0:1789466119.301661:0:201493:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466119.301664:0:201493:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466119.301666:0:201493:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1789466119.301670:0:201493:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 17179870928, transno 0, xid 1876386402356352 00010000:00000001:1.0:1789466119.301673:0:201493:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:1.0:1789466119.301680:0:201493:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189f480 x1876386402356352/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:615/0 lens 224/224 e 0 to 0 dl 1789466130 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466119.301684:0:201490:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1789466119.301688:0:201490:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1789466119.301697:0:201490:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3510 00000100:00000040:3.0:1789466119.301702:0:201490:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967f5cc5000 : new rpc_count 1 00000100:00000001:3.0:1789466119.301705:0:201490:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528264192 : -130395181287424 : ffff89680189f800) 00000100:00000040:3.0:1789466119.301713:0:201490:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189f800 x1876386402356480/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:615/0 lens 224/0 e 0 to 0 dl 1789466130 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466119.301724:0:201490:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466119.301729:0:201490:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657cc40. 00000100:00000001:3.0:1789466119.301733:0:201490:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1789466119.301735:0:201490:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967c657cc40. 00000100:00100000:3.0:1789466119.301740:0:201490:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402356480:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:3.0:1789466119.301744:0:201490:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402356480 00000020:00000001:3.0:1789466119.301748:0:201490:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466119.301751:0:201490:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1789466119.301754:0:201490:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466119.301757:0:201490:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1789466119.301913:0:201490:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:3.0:1789466119.301917:0:201490:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1789466119.301930:0:201490:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1789466119.301934:0:201490:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1789466119.301935:0:201490:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1789466119.301939:0:201490:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466119.301943:0:201490:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00080000:2.0:1789466119.301943:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00000001:3.0:1789466119.301949:0:201490:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1789466119.301953:0:201490:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1789466119.301958:0:201490:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c745a000. 02000000:00000001:3.0:1789466119.301962:0:201490:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1789466119.301963:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:3.0:1789466119.301968:0:201490:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1789466119.301972:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000020:00000001:3.0:1789466119.301973:0:201490:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1789466119.301977:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00010000:00000040:3.0:1789466119.301979:0:201490:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 12884913986, transno 0, xid 1876386402356480 00010000:00000001:3.0:1789466119.301985:0:201490:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:3.0:1789466119.301993:0:201490:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189f800 x1876386402356480/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:615/0 lens 224/224 e 0 to 0 dl 1789466130 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789466119.302003:0:201490:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1789466119.302006:0:201490:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789466119.302011:0:201490:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=28 v=5 (1 1 1 1) 00000100:00000001:3.0:1789466119.302017:0:201490:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466119.302022:0:201490:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:3.0:1789466119.302025:0:201490:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:3.0:1789466119.302028:0:201490:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1789466119.302030:0:201490:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466119.302035:0:201490:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789466119.302065:0:201490:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:3.0:1789466119.302073:0:201490:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed93abb0. 00000100:00000200:3.0:1789466119.302080:0:201490:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402356480, offset 224 00000400:00000200:3.0:1789466119.302088:0:201490:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:3.0:1789466119.302103:0:201490:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:3.0:1789466119.302113:0:201490:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93561:93561:256:4294967295] 192.168.203.42@tcp LPNI seq info [93557:93557:8:4294967295] 00000400:00000200:3.0:1789466119.302128:0:201490:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:3.0:1789466119.302139:0:201490:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:3.0:1789466119.302151:0:201490:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966c9dfe400. 00010000:00000001:1.0:1789466119.302154:0:201493:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1789466119.302157:0:201493:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789466119.302163:0:201493:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=28 v=5 (1 1 1 1) 00000100:00000001:1.0:1789466119.302170:0:201493:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1789466119.302172:0:201490:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000100:00000040:1.0:1789466119.302179:0:201493:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 6 to 192.168.203.42@tcp 00000100:00000001:1.0:1789466119.302183:0:201493:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:1.0:1789466119.302186:0:201493:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1789466119.302188:0:201493:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466119.302192:0:201493:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466119.302196:0:201493:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000800:00000200:3.0:1789466119.302203:0:201490:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000400:00000010:1.0:1789466119.302207:0:201493:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351550. 00000800:00000200:3.0:1789466119.302211:0:201490:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966c9dfe400 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:1.0:1789466119.302217:0:201493:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402356352, offset 224 00000100:00000001:3.0:1789466119.302225:0:201490:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789466119.302230:0:201493:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:1.0:1789466119.302244:0:201493:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:1.0:1789466119.302251:0:201493:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93562:93562:255:4294967295] 192.168.203.42@tcp LPNI seq info [93558:93558:7:4294967295] 00000400:00000200:1.0:1789466119.302265:0:201493:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:1.0:1789466119.302271:0:201493:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:1.0:1789466119.302276:0:201493:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966c3cad700. 00000800:00000200:1.0:1789466119.302283:0:201493:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:1.0:1789466119.302305:0:201493:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:1.0:1789466119.302309:0:201493:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966c3cad700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789466119.302314:0:201493:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466119.302318:0:201493:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00010000:00000001:1.0:1789466119.302322:0:201493:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:1.0:1789466119.302324:0:201493:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466119.302329:0:201493:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1789466119.302336:0:201493:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189f480 x1876386402356352/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:615/0 lens 224/224 e 0 to 0 dl 1789466130 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466119.302350:0:201493:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402356352:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 761us (1341us total) trans 0 rc 0/0 00000100:00100000:1.0:1789466119.302360:0:201493:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3509 00000100:00000040:1.0:1789466119.302364:0:201493:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967c3b5d800 : new rpc_count 0 00000100:00000001:1.0:1789466119.302366:0:201493:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1789466119.302367:0:201493:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1789466119.302373:0:201493:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d832ab80. 00000020:00000010:1.0:1789466119.302378:0:201493:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967f0ef3640. 00000020:00000010:1.0:1789466119.302383:0:201493:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967c5c27a00. 00000020:00000040:1.0:1789466119.302389:0:201493:0:(genops.c:879:class_export_put()) PUTting export ffff8967c3b5d800 : new refcount 6 00000100:00000001:1.0:1789466119.302393:0:201493:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1789466119.302565:0:117758:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789466119.302569:0:117758:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966c9dfe400. 00000400:00000200:2.0:1789466119.302574:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466119.302579:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:2.0:1789466119.302585:0:117758:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed93abb0 00000400:00000010:2.0:1789466119.302588:0:117758:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed93abb0. 00000100:00000001:2.0:1789466119.302592:0:117758:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:2.0:1789466119.302600:0:117758:0:(events.c:402:reply_out_callback()) Process leaving 00000800:00000200:2.0:1789466119.302665:0:117758:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789466119.302668:0:117758:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966c3cad700. 00000400:00000200:2.0:1789466119.302672:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466119.302675:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:2.0:1789466119.302678:0:117758:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351550 00000400:00000010:2.0:1789466119.302680:0:117758:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351550. 00000100:00000001:2.0:1789466119.302683:0:117758:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:2.0:1789466119.302684:0:117758:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00000040:3.0:1789466119.303986:0:201490:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:3.0:1789466119.303992:0:201490:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:3.0:1789466119.303995:0:201490:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466119.303998:0:201490:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1789466119.304009:0:201490:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189f800 x1876386402356480/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:615/0 lens 224/224 e 0 to 0 dl 1789466130 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789466119.304020:0:201490:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253752:x1876386402356480:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 2294us (2865us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466119.304031:0:201490:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3510 00000100:00000040:3.0:1789466119.305412:0:201490:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967f5cc5000 : new rpc_count 0 00000100:00000001:3.0:1789466119.305423:0:201490:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1789466119.305437:0:201490:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1789466119.305446:0:201490:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d832ac00. 00000020:00000010:3.0:1789466119.305453:0:201490:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967f0ef3898. 00000020:00000010:3.0:1789466119.305457:0:201490:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967c5c26e00. 00000020:00000040:3.0:1789466119.305465:0:201490:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5cc5000 : new refcount 6 00000100:00000001:3.0:1789466119.305469:0:201490:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466119.371089:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.371114:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466119.371120:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.371123:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466119.371137:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466119.371162:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 520 into portal 26 MB=0x6aa9040f94d80 00000400:00000100:2.0:1789466119.371172:0:117756:0:(lib-move.c:4350:lnet_parse_put()) Dropping PUT from 12345-192.168.203.42@tcp portal 26 match 1876386402356608 offset 0 length 520: 4 00000400:00000200:2.0:1789466119.371179:0:117756:0:(lib-msg.c:1116:lnet_is_health_check()) Msg ffff8967e7afa420 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:2.0:1789466119.371182:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1789466119.371365:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.371377:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466119.371381:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466119.371383:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.911148:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466119.911163:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466119.911165:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466119.911170:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466119.911173:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466119.911177:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466119.911180:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466120.167178:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466120.167185:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466120.167213:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466120.167216:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466120.167219:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1789466120.552133:0:117755:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f3702d80] -> 12345-192.168.203.142@tcp (6) 00000800:00000200:3.0:1789466120.552142:0:117755:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.142@tcp ip 192.168.203.142:988 00000800:00000200:3.0:1789466120.552147:0:117755:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966c6137000 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:2.0:1789466120.552544:0:117757:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1789466120.552550:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466120.552561:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466120.552566:0:117757:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466120.552568:0:117757:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466120.935129:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466120.935143:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466120.935145:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466120.935150:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466120.935153:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466120.935157:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466120.935160:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466121.191176:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466121.191183:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466121.192712:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466121.192718:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466121.192721:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466121.959163:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466121.959178:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466121.959179:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466121.959183:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466121.959185:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466121.959190:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466121.959192:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466122.215171:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466122.215178:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466122.215206:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466122.215209:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466122.215212:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466122.983148:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466122.983162:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466122.983165:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466122.983170:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466122.983172:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466122.983176:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466122.983179:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466123.239196:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466123.239217:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466123.239321:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789466123.239324:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789466123.239327:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789466123.943722:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466123.943731:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466123.943734:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466123.943737:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466123.943739:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466123.943742:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466123.943745:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466123.943748:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789466123.943750:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789466123.943753:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1789466123.943758:0:134719:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18377) 00000100:00000001:1.0:1789466124.007155:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466124.007776:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466124.007781:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466124.007785:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466124.007788:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466124.007807:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466124.007810:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466124.263299:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466124.263306:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466124.263336:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466124.263339:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466124.263342:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:2.0:1789466124.422003:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466124.422023:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789466124.422027:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466124.422030:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466124.422056:0:117756:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:2.0:1789466124.422075:0:117756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94e80 00000400:00000200:2.0:1789466124.422098:0:117756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 6944 00000800:00000001:2.0:1789466124.422105:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466124.422114:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789466124.422118:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789466124.422122:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:2.0:1789466124.422130:0:117756:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:2.0:1789466124.422133:0:117756:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:2.0:1789466124.422140:0:117756:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89680189d500. 00000100:00000040:2.0:1789466124.422144:0:117756:0:(events.c:341:request_in_callback()) incoming req@ffff89680189d500 x1876386402356864 msgsize 224 00000100:00100000:2.0:1789466124.422148:0:117756:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:2.0:1789466124.422165:0:117756:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:2.0:1789466124.422174:0:117756:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789466124.422177:0:117756:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466124.422730:0:121174:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1789466124.422735:0:121174:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402356864 02000000:00000001:3.0:1789466124.422739:0:121174:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1789466124.422742:0:121174:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789466124.422751:0:121174:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789466124.422756:0:121174:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789466124.422760:0:121174:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402356864 00000020:00000001:3.0:1789466124.422764:0:121174:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1789466124.422766:0:121174:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b8705ac 00000020:00000001:3.0:1789466124.422769:0:121174:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1789466124.422772:0:121174:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967f5cc5000 refcount=7 00000020:00000001:3.0:1789466124.422775:0:121174:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000020:00000001:3.0:1789466124.422777:0:121174:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613678331285504 : -130395378266112 : ffff8967f5cc5000) 00000100:00000001:3.0:1789466124.422781:0:121174:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466124.422785:0:121174:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657cc40. 00000020:00000010:3.0:1789466124.422790:0:121174:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967f014f640. 00000020:00000040:3.0:1789466124.422795:0:121174:0:(obd_config.c:995:class_incref()) incref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00000100:00000001:3.0:1789466124.422816:0:121174:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:3.0:1789466124.422823:0:121174:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967f5dd6800. 00000020:00000010:3.0:1789466124.422828:0:121174:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d3614f00. 00000020:00000010:3.0:1789466124.422833:0:121174:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8966c68cfc80. 00000100:00000040:3.0:1789466124.422843:0:121174:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00000001:3.0:1789466124.422846:0:121174:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1789466124.422848:0:121174:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1789466124.422850:0:121174:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466124.422857:0:121174:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1789466124.422860:0:121174:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1789466124.422862:0:121174:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466124.422879:0:121174:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789466124.422891:0:121174:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1789466124.422893:0:121174:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1789466124.422899:0:121174:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3511 00000100:00000040:3.0:1789466124.422903:0:121174:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967f5cc5000 : new rpc_count 1 00000100:00000001:3.0:1789466124.422905:0:121174:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613678528255232 : -130395181296384 : ffff89680189d500) 00000100:00000040:3.0:1789466124.422913:0:121174:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff89680189d500 x1876386402356864/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:620/0 lens 224/0 e 0 to 0 dl 1789466135 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789466124.422922:0:121174:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1789466124.422932:0:121174:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967c657cbc0. 00000100:00000001:3.0:1789466124.422934:0:121174:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1789466124.422937:0:121174:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967c657cbc0. 00000100:00100000:3.0:1789466124.422942:0:121174:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff89680189d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402356864:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:3.0:1789466124.422945:0:121174:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402356864 00000020:00000001:3.0:1789466124.422949:0:121174:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1789466124.422952:0:121174:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1789466124.422954:0:121174:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466124.422956:0:121174:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1789466124.422958:0:121174:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:3.0:1789466124.422961:0:121174:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1789466124.422964:0:121174:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1789466124.422966:0:121174:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1789466124.422968:0:121174:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1789466124.422971:0:121174:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466124.422973:0:121174:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1789466124.422978:0:121174:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1789466124.422981:0:121174:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1789466124.422986:0:121174:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967c7459400. 02000000:00000001:3.0:1789466124.422988:0:121174:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466124.422991:0:121174:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466124.422993:0:121174:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1789466124.422996:0:121174:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 12884913986, transno 0, xid 1876386402356864 00010000:00000001:3.0:1789466124.422999:0:121174:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00000100:00080000:0.0:1789466124.423004:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00000200:3.0:1789466124.423005:0:121174:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff89680189d500 x1876386402356864/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:620/0 lens 224/224 e 0 to 0 dl 1789466135 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1789466124.423011:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0001 (ffff8966c4f899c0) now 6 - evictor 00010000:00000001:3.0:1789466124.423017:0:121174:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1789466124.423021:0:121174:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789466124.423026:0:121174:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=33 v=5 (1 1 1 1) 00000100:00000001:3.0:1789466124.423030:0:121174:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1789466124.423033:0:121174:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:3.0:1789466124.423060:0:121174:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:3.0:1789466124.423064:0:121174:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1789466124.423074:0:121174:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789466124.423077:0:121174:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789466124.423081:0:121174:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:3.0:1789466124.423085:0:121174:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8967ed93a220. 00000100:00000200:3.0:1789466124.423091:0:121174:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402356864, offset 224 00000400:00000200:3.0:1789466124.423098:0:121174:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:3.0:1789466124.423109:0:121174:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:3.0:1789466124.423116:0:121174:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93563:93563:256:4294967295] 192.168.203.42@tcp LPNI seq info [93559:93559:8:4294967295] 00000400:00000200:3.0:1789466124.423127:0:121174:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:3.0:1789466124.423132:0:121174:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:3.0:1789466124.423137:0:121174:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966c9dfeb00. 00000800:00000200:3.0:1789466124.423143:0:121174:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:3.0:1789466124.423150:0:121174:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:3.0:1789466124.423155:0:121174:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966c9dfeb00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789466124.423172:0:121174:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789466124.423186:0:121174:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:3.0:1789466124.423188:0:121174:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:3.0:1789466124.423190:0:121174:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466124.423192:0:121174:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1789466124.423197:0:121174:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff89680189d500 x1876386402356864/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:620/0 lens 224/224 e 0 to 0 dl 1789466135 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789466124.423216:0:121174:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff89680189d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402356864:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 291us (1069us total) trans 0 rc 0/0 00000100:00100000:3.0:1789466124.423223:0:121174:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3511 00000100:00000040:3.0:1789466124.423227:0:121174:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967f5cc5000 : new rpc_count 0 00000100:00000001:3.0:1789466124.423229:0:121174:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1789466124.423230:0:121174:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1789466124.423235:0:121174:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d3614f00. 00000020:00000010:3.0:1789466124.423239:0:121174:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8966c68cfc80. 00000020:00000010:3.0:1789466124.423242:0:121174:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967f5dd6800. 00000020:00000040:3.0:1789466124.423252:0:121174:0:(genops.c:879:class_export_put()) PUTting export ffff8967f5cc5000 : new refcount 6 00000100:00000001:3.0:1789466124.423256:0:121174:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1789466124.423351:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1789466124.423359:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1789466124.423361:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1789466124.423363:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1789466124.423368:0:117758:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp) <- 192.168.203.42@tcp : PUT - for me 00000400:00000200:0.0:1789466124.423378:0:117758:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.42@tcp of length 224 into portal 28 MB=0x6aa9040f94e00 00000400:00000200:0.0:1789466124.423384:0:117758:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1c from 12345-192.168.203.42@tcp of length 224/224 into md 0x53ed5 [8] + 7168 00000800:00000001:0.0:1789466124.423401:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1789466124.423407:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1789466124.423410:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1789466124.423413:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000100:00000001:0.0:1789466124.423418:0:117758:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:0.0:1789466124.423419:0:117758:0:(events.c:300:request_in_callback()) event type 2, status 0, service ost 00000100:00000010:0.0:1789466124.423423:0:117758:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8967cb941180. 00000100:00000040:0.0:1789466124.423426:0:117758:0:(events.c:341:request_in_callback()) incoming req@ffff8967cb941180 x1876386402356736 msgsize 224 00000100:00100000:0.0:1789466124.423430:0:117758:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.42@tcp (source: 12345-192.168.203.42@tcp) 00000100:00000001:0.0:1789466124.423494:0:117758:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000200:0.0:1789466124.423568:0:117758:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1789466124.423573:0:117758:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966c9dfeb00. 00000400:00000200:0.0:1789466124.423577:0:117758:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1789466124.423581:0:117758:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:0.0:1789466124.423585:0:117758:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8967ed93a220 00000400:00000010:0.0:1789466124.423587:0:117758:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8967ed93a220. 00000100:00000001:0.0:1789466124.423591:0:117758:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:0.0:1789466124.423592:0:117758:0:(events.c:402:reply_out_callback()) Process leaving 00000800:00000001:0.0:1789466124.423597:0:117758:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1789466124.423607:0:117758:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466124.424217:0:121172:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1789466124.424221:0:121172:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876386402356736 02000000:00000001:1.0:1789466124.424223:0:121172:0:(sec.c:2511:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1789466124.424225:0:121172:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789466124.424227:0:121172:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789466124.424231:0:121172:0:(sec.c:2568:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789466124.424235:0:121172:0:(service.c:2371:ptlrpc_server_handle_req_in()) got req x1876386402356736 00000020:00000001:1.0:1789466124.424237:0:121172:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1789466124.424238:0:121172:0:(genops.c:804:class_conn2export()) looking for export cookie 0x6da27ab53b86f3e6 00000020:00000001:1.0:1789466124.424240:0:121172:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1789466124.424243:0:121172:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8967c3b5d800 refcount=7 00000020:00000001:1.0:1789466124.424246:0:121172:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000020:00000001:1.0:1789466124.424248:0:121172:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613677490952192 : -130396218599424 : ffff8967c3b5d800) 00000100:00000001:1.0:1789466124.424251:0:121172:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1789466124.424256:0:121172:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967f014fe00. 00000020:00000010:1.0:1789466124.424259:0:121172:0:(genops.c:1111:obd_export_timed_del()) kfreed 'ned': 48 at ffff8967f014f440. 00000020:00000040:1.0:1789466124.424264:0:121172:0:(obd_config.c:995:class_incref()) incref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000001:1.0:1789466124.424278:0:121172:0:(service.c:1358:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:1.0:1789466124.424282:0:121172:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8967c5c27a00. 00000020:00000010:1.0:1789466124.424287:0:121172:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8967d832ab80. 00000020:00000010:1.0:1789466124.424291:0:121172:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8967f0ef3640. 00000100:00000040:1.0:1789466124.424298:0:121172:0:(service.c:1432:ptlrpc_at_set_timer()) armed ost at +6s 00000100:00000001:1.0:1789466124.424301:0:121172:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1789466124.424302:0:121172:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1789466124.424304:0:121172:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466124.424309:0:121172:0:(nrs.c:885:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1789466124.424311:0:121172:0:(nrs.c:893:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1789466124.424312:0:121172:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466124.424326:0:121172:0:(service.c:2450:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466124.424335:0:121172:0:(service.c:2484:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1789466124.424337:0:121172:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1789466124.424348:0:121172:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.42@tcp, seq: 3512 00000100:00000040:1.0:1789466124.424352:0:121172:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export ffff8967c3b5d800 : new rpc_count 1 00000100:00000001:1.0:1789466124.424361:0:121172:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446613677622956416 : -130396086595200 : ffff8967cb941180) 00000100:00000040:1.0:1789466124.424368:0:121172:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@ffff8967cb941180 x1876386402356736/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:620/0 lens 224/0 e 0 to 0 dl 1789466135 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789466124.424377:0:121172:0:(service.c:1268:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1789466124.424380:0:121172:0:(genops.c:1054:obd_export_timed_init()) kmalloced '(*data)': 48 at ffff8967f014f440. 00000100:00000001:1.0:1789466124.424382:0:121172:0:(service.c:1300:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1789466124.424393:0:121172:0:(genops.c:1062:obd_export_timed_fini()) kfreed '*data': 48 at ffff8967f014f440. 00000100:00100000:1.0:1789466124.424398:0:121172:0:(service.c:2557:ptlrpc_server_handle_request()) Handling RPC req@ffff8967cb941180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402356736:12345-192.168.203.42@tcp:400:kworker.0 00000100:00000200:1.0:1789466124.424401:0:121172:0:(service.c:2568:ptlrpc_server_handle_request()) got req 1876386402356736 00000020:00000001:1.0:1789466124.424404:0:121172:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1789466124.424406:0:121172:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1789466124.424408:0:121172:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466124.424410:0:121172:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1789466124.424411:0:121172:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072649832640 : -1059718976 : ffffffffc0d5f8c0) 00000020:00000001:1.0:1789466124.424414:0:121172:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1789466124.424416:0:121172:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1789466124.424418:0:121172:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1789466124.424419:0:121172:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1789466124.424421:0:121172:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466124.424423:0:121172:0:(tgt_handler.c:1163:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1789466124.424425:0:121172:0:(pack_generic.c:297:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1789466124.424427:0:121172:0:(sec.c:2590:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1789466124.424431:0:121172:0:(sec_null.c:300:null_alloc_rs()) kmalloced '(rs)': 520 at ffff8967fe7f2c00. 02000000:00000001:1.0:1789466124.424433:0:121172:0:(sec.c:2627:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466124.424435:0:121172:0:(pack_generic.c:332:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466124.424436:0:121172:0:(tgt_handler.c:1184:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1789466124.424438:0:121172:0:(ldlm_lib.c:3342:target_committed_to_req()) last_committed 17179870928, transno 0, xid 1876386402356736 00010000:00000001:1.0:1789466124.424441:0:121172:0:(ldlm_lib.c:3411:target_send_reply()) Process entered 00010000:00000200:1.0:1789466124.424445:0:121172:0:(ldlm_lib.c:3399:target_send_reply_msg()) @@@ sending reply req@ffff8967cb941180 x1876386402356736/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:620/0 lens 224/224 e 0 to 0 dl 1789466135 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1789466124.424452:0:121172:0:(ldlm_lib.c:3354:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1789466124.424454:0:121172:0:(ldlm_lib.c:3375:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789466124.424463:0:121172:0:(import.c:2196:obd_at_measure()) add 1 to ffff8967f416e1e8 time=33 v=5 (1 1 1 1) 00000100:00000001:1.0:1789466124.424467:0:121172:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1789466124.424469:0:121172:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8967e6e92380 refcount 5 to 192.168.203.42@tcp 00000100:00000001:1.0:1789466124.424471:0:121172:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613678081516416 : -130395628035200 : ffff8967e6e92380) 02000000:00000001:1.0:1789466124.424473:0:121172:0:(sec.c:2641:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1789466124.424475:0:121172:0:(sec.c:2652:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466124.424477:0:121172:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789466124.424479:0:121172:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.42@tcp 00000400:00000010:1.0:1789466124.424483:0:121172:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8966c6351110. 00000100:00000200:1.0:1789466124.424497:0:121172:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 4, xid 1876386402356736, offset 224 00000400:00000200:1.0:1789466124.424504:0:121172:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.203.42@tcp 00000400:00000200:1.0:1789466124.424513:0:121172:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source Specified: 192.168.203.142@tcp to MR: 192.168.203.42@tcp local destination 00000400:00000200:1.0:1789466124.424518:0:121172:0:(lib-move.c:1738:lnet_handle_send()) 192.168.203.142@tcp NI seq info: [93564:93564:256:4294967295] 192.168.203.42@tcp LPNI seq info [93560:93560:8:4294967295] 00000400:00000200:1.0:1789466124.424527:0:121172:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.203.142@tcp(192.168.203.142@tcp:192.168.203.142@tcp) -> 192.168.203.42@tcp(192.168.203.42@tcp:192.168.203.42@tcp) : PUT try# 0 00000800:00000200:1.0:1789466124.424532:0:121172:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.42@tcp 00000800:00000010:1.0:1789466124.424536:0:121172:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8966c3cad000. 00000800:00000200:1.0:1789466124.424541:0:121172:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8966f9f54000] -> 12345-192.168.203.42@tcp (4) 00000800:00000200:1.0:1789466124.424558:0:121172:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.42@tcp ip 192.168.203.42:1023 00000800:00000200:1.0:1789466124.424561:0:121172:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8966c3cad000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789466124.424577:0:121172:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789466124.424580:0:121172:0:(lustre_net.h:1978:ptlrpc_connection_put()) PUT conn=ffff8967e6e92380 refcount 4 to 192.168.203.42@tcp 00010000:00000001:1.0:1789466124.424582:0:121172:0:(ldlm_lib.c:3423:target_send_reply()) Process leaving 00000020:00000001:1.0:1789466124.424584:0:121172:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789466124.424586:0:121172:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1789466124.424590:0:121172:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8967cb941180 x1876386402356736/t0(0) o400->5f8c9c46-c8d9-41fe-8865-a17a1eaf923a@192.168.203.42@tcp:620/0 lens 224/224 e 0 to 0 dl 1789466135 ref 1 fl Interpret:H/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789466124.424599:0:121172:0:(service.c:2615:ptlrpc_server_handle_request()) Handled RPC req@ffff8967cb941180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:5f8c9c46-c8d9-41fe-8865-a17a1eaf923a+7:253751:x1876386402356736:12345-192.168.203.42@tcp:400:kworker.0 Request processed in 221us (1171us total) trans 0 rc 0/0 00000100:00100000:1.0:1789466124.424606:0:121172:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.42@tcp, seq: 3512 00000100:00000040:1.0:1789466124.424609:0:121172:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export ffff8967c3b5d800 : new rpc_count 0 00000100:00000001:1.0:1789466124.424616:0:121172:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1789466124.424618:0:121172:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1789466124.424621:0:121172:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8967d832ab80. 00000020:00000010:1.0:1789466124.424625:0:121172:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8967f0ef3640. 00000020:00000010:1.0:1789466124.424627:0:121172:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8967c5c27a00. 00000020:00000040:1.0:1789466124.424631:0:121172:0:(genops.c:879:class_export_put()) PUTting export ffff8967c3b5d800 : new refcount 6 00000100:00000001:1.0:1789466124.424633:0:121172:0:(service.c:2640:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1789466124.426847:0:117756:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789466124.426857:0:117756:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8966c3cad000. 00000400:00000200:3.0:1789466124.426865:0:117756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789466124.426873:0:117756:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.142@tcp->192.168.203.42@tcp: PUT: OK 00000400:00000200:3.0:1789466124.426881:0:117756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8966c6351110 00000400:00000010:3.0:1789466124.426884:0:117756:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8966c6351110. 00000100:00000001:3.0:1789466124.426889:0:117756:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:3.0:1789466124.426891:0:117756:0:(events.c:402:reply_out_callback()) Process leaving 00000100:00080000:0.0:1789466124.430817:0:119359:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000020:00000040:0.0:1789466124.430826:0:119359:0:(obd_config.c:1023:class_decref()) Decref lustre-OST0000 (ffff8967ffdc0000) now 6 - evictor 00000100:00000040:0.0:1789466124.455255:0:222999:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37000 (cpu 0) has reached its deadline 4313039475, at 4313039872 00000100:00000010:0.0:1789466124.455271:0:222999:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc114e0. 00000100:00000040:2.0:1789466124.455621:0:222031:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37220 (cpu 2) has reached its deadline 4313039475, at 4313039872 00000100:00000010:2.0:1789466124.455628:0:222031:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc11cc0. 00000100:00000040:1.0:1789466124.455653:0:209110:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37110 (cpu 1) has reached its deadline 4313039475, at 4313039872 00000100:00000010:1.0:1789466124.455659:0:209110:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc11600. 00000100:00000040:3.0:1789466124.455865:0:218018:0:(connection.c:75:cpu_latency_work()) work item of ffff8967edc37330 (cpu 3) has reached its deadline 4313039475, at 4313039872 00000100:00000010:3.0:1789466124.455873:0:218018:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8967dfc11180. 00000100:00000001:1.0:1789466125.031569:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466125.031584:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466125.031586:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466125.031591:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466125.031593:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466125.031598:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466125.031601:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466125.287387:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466125.287393:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466125.287498:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466125.287502:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466125.287505:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466126.055760:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466126.055775:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466126.055777:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466126.055782:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466126.055785:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466126.055791:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466126.055793:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466126.311560:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466126.311567:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466126.311598:0:187061:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466126.311602:0:187061:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466126.311604:0:187061:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1789466127.079091:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466127.079109:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466127.079112:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466127.079117:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466127.079120:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466127.079125:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466127.079129:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:3.0:1789466127.127537:0:224884:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3614d00. 00080000:00000010:3.0:1789466127.127551:0:224884:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3614d00. 00000020:00000001:3.0:1789466127.127559:0:224884:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:3.0:1789466127.127569:0:224884:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3614d00. 00080000:00000010:3.0:1789466127.127576:0:224884:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3614d00. 00000020:00000001:3.0:1789466127.127580:0:224884:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:2.0:1789466127.218744:0:1:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967e983da80. 00080000:00000010:2.0:1789466127.218760:0:1:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967e983da80. 00000020:00000001:2.0:1789466127.218770:0:1:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:2.0:1789466127.218781:0:1:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967e983da80. 00080000:00000010:2.0:1789466127.218787:0:1:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967e983da80. 00000020:00000001:2.0:1789466127.218791:0:1:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466127.335556:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466127.335563:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789466127.339556:0:119352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789466127.339567:0:119352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789466127.339574:0:119352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00080000:00000010:0.0:1789466127.345458:0:224899:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3528f80. 00080000:00000010:0.0:1789466127.345474:0:224899:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3528f80. 00000020:00000001:0.0:1789466127.345483:0:224899:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:0.0:1789466127.345492:0:224899:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3528f80. 00080000:00000010:0.0:1789466127.345498:0:224899:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3528f80. 00000020:00000001:0.0:1789466127.345502:0:224899:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:3.0:1789466127.560752:0:224899:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3614f80. 00080000:00000010:3.0:1789466127.560768:0:224899:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3614f80. 00000020:00000001:3.0:1789466127.560778:0:224899:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:3.0:1789466127.560789:0:224899:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967d3614f80. 00080000:00000010:3.0:1789466127.560796:0:224899:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967d3614f80. 00000020:00000001:3.0:1789466127.560801:0:224899:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1789466128.076086:0:224933:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:3.0:1789466128.076093:0:224933:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity test_160j: @@@@@@ FAIL: mount_client on /mnt/lustre2 failed 00000001:00000001:3.0:1789466128.083903:0:224933:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00000100:00000001:1.0:1789466128.104210:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466128.104222:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466128.104224:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466128.104228:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466128.104231:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466128.104236:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466128.104239:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466128.359173:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466128.359182:0:222999:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789466128.359210:0:122432:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789466128.359213:0:122432:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789466128.359216:0:122432:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789466129.063195:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466129.063205:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) 4861e2df-2452-4fd5-a0b1-355ea409d38c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466129.063208:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466129.063211:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466129.063214:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466129.063216:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0000_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466129.063237:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466129.063240:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789466129.063243:0:134719:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789466129.063246:0:134719:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-lwp-OST0001_UUID->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1789466129.063252:0:134719:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (18382) 00000100:00000001:1.0:1789466129.127236:0:117762:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1789466129.127268:0:117762:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789466129.127270:0:117762:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789466129.127275:0:117762:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789466129.127277:0:117762:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789466129.127281:0:117762:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789466129.127284:0:117762:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:2.0:1789466129.217787:0:224945:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967e983d300. 00080000:00000010:2.0:1789466129.217798:0:224945:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967e983d300. 00000020:00000001:2.0:1789466129.217805:0:224945:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:2.0:1789466129.217814:0:224945:0:(osd_handler.c:2598:osd_statfs()) kmalloced '(ksfs)': 120 at ffff8967e983d300. 00080000:00000010:2.0:1789466129.217819:0:224945:0:(osd_handler.c:2639:osd_statfs()) kfreed 'ksfs': 120 at ffff8967e983d300. 00000020:00000001:2.0:1789466129.217823:0:224945:0:(tgt_mount.c:2304:server_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466129.384928:0:222999:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789466129.384938:0:222999:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789466129.384940:0:222999:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789466129.384943:0:222999:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466129.384947:0:222999:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789466129.384950:0:222999:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466129.384952:0:222999:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789466129.384954:0:222999:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789466129.386955:0:119351:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789466129.386961:0:119351:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789466129.386965:0:119351:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 3024 lines, 3024 kept, 0 dropped, 0 bad.