00010000:00000001:2.0F:1789036423.742978:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036423.743002:0:274228:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1789036423.743004:0:274228:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1789036423.743007:0:274228:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036423.743012:0:274228:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1789036423.743014:0:274228:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036423.743016:0:274228:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036423.743019:0:274228:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036423.743023:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036423.743025:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0F:1789036423.743200:0:248124:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036423.743210:0:248124:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036423.743213:0:248124:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036424.766965:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036424.766971:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036424.766977:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036424.766979:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036424.767067:0:242419:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036424.767071:0:242419:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036424.767074:0:242419:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036425.790967:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036425.790973:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036425.791037:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036425.791038:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0F:1789036425.791166:0:236342:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789036425.791179:0:236342:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789036425.791182:0:236342:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1789036426.304496:0:252176:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712cb30 (cpu 3) has reached its deadline 4307507414, at 4307507457 00000100:00000010:3.0:1789036426.304510:0:252176:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8b9d5bf60. 00000100:00000040:2.0:1789036426.305096:0:274228:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712ca20 (cpu 2) has reached its deadline 4307507414, at 4307507458 00000100:00000010:2.0:1789036426.305103:0:274228:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8b9d5bf00. 00010000:00000001:2.0:1789036426.814945:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036426.814952:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036426.814957:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036426.814959:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036426.815054:0:252249:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036426.815058:0:252249:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036426.815061:0:252249:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0F:1789036427.583348:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036427.583365:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789036427.583372:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f7a300. 00000020:00000040:1.0:1789036427.583378:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d8861f3800 refcount=4 obd=MGC192.168.204.121@tcp 00000100:00000001:1.0:1789036427.583382:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036427.583385:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036427.583390:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036427.583396:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2ab00. 00000100:00000001:1.0:1789036427.583405:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583417:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036427.583429:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036427.583433:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583439:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041a5e0ff] to pc [ptlrpcd_00_00+0] req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.583463:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583465:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036427.583469:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789036427.583472:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f7b100. 00000020:00000040:1.0:1789036427.583474:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d887edf000 refcount=6 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000100:00000001:1.0:1789036427.583477:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036427.583478:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036427.583480:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036427.583483:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2a800. 00000100:00000001:1.0:1789036427.583486:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583490:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036427.583498:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036427.583500:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583504:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0348126] to pc [ptlrpcd_00_01+1] req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.583512:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583519:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036427.583522:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789036427.583525:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f78700. 00000020:00000040:1.0:1789036427.583528:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d887edb000 refcount=4 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000100:00000001:1.0:1789036427.583530:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036427.583531:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036427.583533:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036427.583536:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2a200. 00000100:00000001:1.0:1789036427.583540:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583555:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036427.583563:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036427.583565:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583569:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a80e34e] to pc [ptlrpcd_00_00+0] req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.583576:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583578:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036427.583581:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789036427.583584:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f7bb80. 00000020:00000040:1.0:1789036427.583586:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d887d65000 refcount=4 obd=lustre-OST0000-osc-ffff97d8b9387800 00000100:00000001:1.0:1789036427.583587:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036427.583589:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036427.583590:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036427.583593:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2af00. 00000100:00000001:1.0:1789036427.583596:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583599:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036427.583607:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036427.583609:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583613:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000858a87ca] to pc [ptlrpcd_00_01+1] req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.583630:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.583633:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1789036427.583636:0:234282:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1789036427.583641:0:234282:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12846) 00000100:00000001:1.0:1789036427.583851:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.583855:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036427.583857:0:2393:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036427.583863:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.583870:0:2393:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036427.583873:0:2393:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.583876:0:2393:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036427.583879:0:2393:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036427.583881:0:2393:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036427.583885:0:2393:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f7a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2393:1875927346551424:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036427.583889:0:2393:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036427.583894:0:2393:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036427.583895:0:2393:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.583898:0:2393:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036427.583902:0:2393:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8a7c61800. 02000000:00000001:1.0:1789036427.583903:0:2393:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036427.583910:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036427.583914:0:2393:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903e580. 00000400:00000010:1.0:1789036427.583921:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d4c8. 00000100:00000200:1.0:1789036427.583927:0:2393:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346551424, portal 25 00000100:00000001:1.0:1789036427.583929:0:2393:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036427.583931:0:2393:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267937024 : -114518441614592 : ffff97d898f7a300) 00000100:00000040:1.0:1789036427.583936:0:2393:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.583943:0:2393:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036427.583952:0:2393:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036427.583954:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62daa0. 00000100:00000200:1.0:1789036427.583959:0:2393:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875927346551424, offset 0 00000400:00000200:1.0:1789036427.583965:0:2393:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036427.583974:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.583982:0:2393:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036427.583987:0:2393:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036427.583992:0:2393:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036427.583995:0:2393:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036427.583998:0:2393:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584002:0:2393:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556978:556978:256:4294967295] 192.168.204.121@tcp LPNI seq info [556932:556932:8:4294967295] 00000400:00000200:1.0:1789036427.584006:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584016:0:2393:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036427.584023:0:2393:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036427.584028:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2a100. 00000800:00000200:1.0:1789036427.584035:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036427.584042:0:2393:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036427.584046:0:2393:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2a100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036427.584064:0:2393:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036427.584069:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036427.584071:0:2393:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 2 is 4307512534 00000100:00000010:1.0:1789036427.584077:0:2393:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d887b737e0. 00000100:00000001:1.0:1789036427.584083:0:2393:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1789036427.584087:0:2393:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 3 is 4307512534 00000100:00000010:1.0:1789036427.584090:0:2393:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d887b738a0. 00000100:00000001:1.0:1789036427.584093:0:2393:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1789036427.584096:0:2393:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.584098:0:2393:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584101:0:2393:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036427.584106:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.584122:0:2393:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036427.584124:0:2393:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.584126:0:2393:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036427.584128:0:2393:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036427.584130:0:2393:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036427.584133:0:2393:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f78700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346551680:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036427.584135:0:2393:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036427.584138:0:2393:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036427.584140:0:2393:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.584142:0:2393:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036427.584144:0:2393:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8a7c61200. 02000000:00000001:1.0:1789036427.584146:0:2393:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036427.584150:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036427.584153:0:2393:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903ed10. 00000400:00000010:1.0:1789036427.584157:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d5d8. 00000100:00000200:1.0:1789036427.584160:0:2393:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346551680, portal 10 00000100:00000001:1.0:1789036427.584162:0:2393:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036427.584164:0:2393:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267929856 : -114518441621760 : ffff97d898f78700) 00000100:00000040:1.0:1789036427.584170:0:2393:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.584176:0:2393:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036427.584178:0:2393:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036427.584180:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d550. 00000100:00000200:1.0:1789036427.584183:0:2393:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346551680, offset 0 00000400:00000200:1.0:1789036427.584187:0:2393:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584190:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584194:0:2393:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036427.584196:0:2393:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036427.584199:0:2393:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036427.584201:0:2393:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036427.584203:0:2393:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584213:0:2393:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556979:556979:255:4294967295] 192.168.204.121@tcp LPNI seq info [556933:556933:7:4294967295] 00000400:00000200:1.0:1789036427.584217:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584223:0:2393:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036427.584227:0:2393:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036427.584231:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2a600. 00000800:00000200:1.0:1789036427.584234:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036427.584238:0:2393:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036427.584240:0:2393:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2a600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036427.584242:0:2393:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.584245:0:2393:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: using a timeout of 19615 usecs (20 jiffies) 00000400:00000200:1.0:1789036427.584248:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036427.584250:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307508756 00000100:00000001:1.0:1789036427.584252:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307512534 00000100:00000001:1.0:1789036427.584254:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307508756 00000100:00000001:1.0:1789036427.584256:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307512534 00000100:00000001:1.0:1789036427.584258:0:2393:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.584260:0:2393:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584262:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584264:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.584271:0:2393:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036427.584273:0:2393:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1789036427.584275:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.584276:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036427.584278:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036427.584280:0:2393:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00000001:1.0:1789036427.584282:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.584283:0:2393:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036427.584285:0:2393:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.584286:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.584287:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036427.584288:0:2393:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036427.584293:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.584305:0:2393:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036427.584307:0:2393:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.584309:0:2393:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036427.584311:0:2393:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036427.584312:0:2393:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036427.584315:0:2393:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f7b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346551552:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036427.584319:0:2393:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036427.584321:0:2393:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036427.584323:0:2393:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.584324:0:2393:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036427.584327:0:2393:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8a7c61400. 02000000:00000001:1.0:1789036427.584329:0:2393:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036427.584332:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036427.584335:0:2393:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903e840. 00000400:00000010:1.0:1789036427.584338:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d880. 00000100:00000200:1.0:1789036427.584341:0:2393:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346551552, portal 10 00000100:00000001:1.0:1789036427.584343:0:2393:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036427.584345:0:2393:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267940608 : -114518441611008 : ffff97d898f7b100) 00000100:00000040:1.0:1789036427.584349:0:2393:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.584355:0:2393:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036427.584357:0:2393:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036427.584359:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d198. 00000100:00000200:1.0:1789036427.584361:0:2393:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346551552, offset 0 00000400:00000200:1.0:1789036427.584365:0:2393:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584369:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584372:0:2393:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036427.584375:0:2393:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036427.584377:0:2393:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036427.584379:0:2393:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036427.584381:0:2393:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584389:0:2393:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556980:556980:254:4294967295] 192.168.204.121@tcp LPNI seq info [556934:556934:6:4294967295] 00000400:00000200:1.0:1789036427.584393:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584400:0:2393:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036427.584403:0:2393:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036427.584407:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2ac00. 00000800:00000200:1.0:1789036427.584411:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036427.584415:0:2393:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036427.584417:0:2393:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2ac00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036427.584420:0:2393:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.584423:0:2393:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: using a timeout of 25706 usecs (26 jiffies) 00000400:00000200:1.0:1789036427.584426:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036427.584428:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307508763 00000100:00000001:1.0:1789036427.584430:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307512534 00000100:00000001:1.0:1789036427.584432:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307508763 00000100:00000001:1.0:1789036427.584434:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307512534 00000100:00000001:1.0:1789036427.584436:0:2393:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.584438:0:2393:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584440:0:2393:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036427.584444:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.584450:0:2393:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036427.584452:0:2393:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.584454:0:2393:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036427.584456:0:2393:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036427.584458:0:2393:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036427.584461:0:2393:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f7bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346551808:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036427.584464:0:2393:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036427.584466:0:2393:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036427.584468:0:2393:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.584475:0:2393:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036427.584478:0:2393:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8a7c60a00. 02000000:00000001:1.0:1789036427.584480:0:2393:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036427.584484:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036427.584486:0:2393:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903e210. 00000400:00000010:1.0:1789036427.584490:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d000. 00000100:00000200:1.0:1789036427.584493:0:2393:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346551808, portal 4 00000100:00000001:1.0:1789036427.584495:0:2393:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036427.584496:0:2393:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267943296 : -114518441608320 : ffff97d898f7bb80) 00000100:00000040:1.0:1789036427.584502:0:2393:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.584509:0:2393:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036427.584511:0:2393:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036427.584514:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62dee0. 00000100:00000200:1.0:1789036427.584516:0:2393:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875927346551808, offset 0 00000400:00000200:1.0:1789036427.584520:0:2393:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584524:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584527:0:2393:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036427.584530:0:2393:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036427.584532:0:2393:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036427.584534:0:2393:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036427.584536:0:2393:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036427.584539:0:2393:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556981:556981:253:4294967295] 192.168.204.121@tcp LPNI seq info [556935:556935:5:4294967295] 00000400:00000200:1.0:1789036427.584553:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036427.584560:0:2393:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036427.584564:0:2393:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036427.584568:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2ae00. 00000800:00000200:1.0:1789036427.584571:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036427.584575:0:2393:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036427.584578:0:2393:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2ae00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036427.584581:0:2393:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.584588:0:2393:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: using a timeout of 102847 usecs (103 jiffies) 00000400:00000200:1.0:1789036427.584592:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036427.584594:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307508840 00000100:00000001:1.0:1789036427.584596:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307512534 00000100:00000001:1.0:1789036427.584598:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307508840 00000100:00000001:1.0:1789036427.584600:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307512534 00000100:00000001:1.0:1789036427.584602:0:2393:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.584604:0:2393:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584606:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584608:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.584618:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.584620:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1789036427.586506:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036427.586514:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2a100. 00000400:00000200:3.0:1789036427.586520:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036427.586527:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036427.586533:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62daa0 00000400:00000010:3.0:1789036427.586536:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62daa0. 00000100:00000001:3.0:1789036427.586541:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036427.586560:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586572:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036427.586576:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586584:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036427.586586:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036427.586597:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036427.586600:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2a600. 00000400:00000200:3.0:1789036427.586603:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036427.586606:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036427.586609:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d550 00000400:00000010:3.0:1789036427.586611:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d550. 00000100:00000001:3.0:1789036427.586613:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036427.586617:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586625:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036427.586628:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586635:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036427.586636:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036427.586652:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036427.586655:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2ac00. 00000400:00000200:3.0:1789036427.586657:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036427.586661:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036427.586664:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d198 00000400:00000010:3.0:1789036427.586665:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d198. 00000100:00000001:3.0:1789036427.586668:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036427.586672:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586693:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036427.586696:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586704:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036427.586705:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036427.586734:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036427.586736:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2ae00. 00000400:00000200:3.0:1789036427.586739:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036427.586742:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036427.586745:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62dee0 00000400:00000010:3.0:1789036427.586747:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62dee0. 00000100:00000001:3.0:1789036427.586749:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036427.586754:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586762:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036427.586764:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036427.586770:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036427.586772:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1789036427.602314:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602522:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036427.602527:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602529:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.602538:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602558:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036427.602567:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602573:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602578:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602583:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1d9b80 00000400:00000200:2.0:1789036427.602589:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602594:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x53924d [1] + 224 00000800:00000001:2.0:1789036427.602602:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602613:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.602617:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036427.602622:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036427.602629:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036427.602638:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036427.602651:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036427.602671:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036427.602681:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602685:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036427.602687:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602689:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.602692:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602696:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036427.602701:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602704:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602707:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602710:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1d9b00 00000400:00000200:2.0:1789036427.602714:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.602717:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x539255 [1] + 224 00000800:00000001:2.0:1789036427.602722:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602729:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.602731:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036427.602735:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036427.602738:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036427.602743:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036427.602752:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036427.602760:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036427.602770:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.602772:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.602934:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.602938:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1789036427.603194:0:2393:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036427.603196:0:2393:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036427.603200:0:2393:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903e840. 00000400:00000200:1.0:1789036427.603213:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d880 00000400:00000010:1.0:1789036427.603216:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d880. 00000100:00000001:1.0:1789036427.603219:0:2393:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036427.603225:0:2393:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036427.603234:0:2393:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.603239:0:2393:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036427.603241:0:2393:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.603244:0:2393:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036427.603247:0:2393:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036427.603249:0:2393:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036427.603251:0:2393:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.603254:0:2393:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036427.603263:0:2393:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edf460 time=88 v=60 (60 61 61 62) 00000100:00001000:1.0:1789036427.603267:0:2393:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edf418 time=124 v=5 (1 1 0 0) 00000100:00000001:1.0:1789036427.603279:0:2393:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036427.603281:0:2393:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036427.603285:0:2393:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1789036427.603287:0:2393:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.603291:0:2393:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1789036427.603294:0:2393:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff97d8b9387800: skip recheck: last_committed 30064771460 00000100:00000001:1.0:1789036427.603296:0:2393:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1789036427.603298:0:2393:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.603303:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.603311:0:2393:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.603314:0:2393:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036427.603326:0:2393:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036427.603330:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036427.603339:0:2393:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f7b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346551552:192.168.204.121@tcp:400:kworker.0 00000400:00000001:1.0:1789036427.603346:0:2393:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036427.603348:0:2393:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036427.603350:0:2393:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903ed10. 00000400:00000200:1.0:1789036427.603353:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d5d8 00000400:00000010:1.0:1789036427.603354:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d5d8. 00000100:00000001:1.0:1789036427.603357:0:2393:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036427.603360:0:2393:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036427.604578:0:2393:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.604587:0:2393:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036427.604590:0:2393:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.604593:0:2393:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036427.604595:0:2393:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036427.604597:0:2393:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036427.604599:0:2393:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.604611:0:2393:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036427.604618:0:2393:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edb460 time=88 v=60 (60 61 61 62) 00000100:00001000:1.0:1789036427.604622:0:2393:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edb418 time=101 v=5 (1 1 0 0) 00000100:00000001:1.0:1789036427.604624:0:2393:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036427.604626:0:2393:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036427.604629:0:2393:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1789036427.604633:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1789036427.604641:0:2393:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.604644:0:2393:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1789036427.604646:0:2393:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff97d8b9387800: skip recheck: last_committed 21474836540 00000100:00000001:1.0:1789036427.604648:0:2393:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1789036427.604650:0:2393:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.604654:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.604660:0:2393:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.604663:0:2393:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036427.604665:0:2393:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036427.604668:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036427.604677:0:2393:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f78700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346551680:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036427.604681:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.604684:0:2393:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036427.604689:0:2393:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f7b100 x1875927346551552/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.604695:0:2393:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036427.604698:0:2393:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036427.604702:0:2393:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8a7c61400. 02000000:00000001:1.0:1789036427.604705:0:2393:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036427.604708:0:2393:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036427.604710:0:2393:0:(genops.c:1222:class_import_put()) import ffff97d887edf000 refcount=5 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000800:00000001:2.0:1789036427.605190:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605204:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036427.605207:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605210:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.605217:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605282:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036427.605291:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605296:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605302:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605306:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 4 MB=0x6aa255f1d9c00 00000400:00000200:2.0:1789036427.605310:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605404:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.121@tcp of length 224/224 into md 0x53925d [1] + 224 00000800:00000001:2.0:1789036427.605412:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605417:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.605420:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036427.605425:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036427.605432:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036427.605438:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036427.605448:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036427.605456:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036427.605466:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605468:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036427.605470:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605472:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.605476:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605479:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036427.605483:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605485:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605488:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605498:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 25 MB=0x6aa255f1d9a80 00000400:00000200:2.0:1789036427.605502:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036427.605506:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.121@tcp of length 224/224 into md 0x539245 [1] + 224 00000800:00000001:2.0:1789036427.605511:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605517:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036427.605519:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036427.605522:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036427.605526:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036427.605531:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036427.605539:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036427.605558:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036427.605562:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036427.605565:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1789036427.606743:0:2393:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036427.606748:0:2393:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2a800. 02000000:00000001:1.0:1789036427.606753:0:2393:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036427.606755:0:2393:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036427.606758:0:2393:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f7b100. 00000100:00000001:1.0:1789036427.606761:0:2393:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036427.606763:0:2393:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.606767:0:2393:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036427.606774:0:2393:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f78700 x1875927346551680/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036512 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.606781:0:2393:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036427.606783:0:2393:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036427.606785:0:2393:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8a7c61200. 02000000:00000001:1.0:1789036427.606788:0:2393:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036427.606789:0:2393:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036427.606791:0:2393:0:(genops.c:1222:class_import_put()) import ffff97d887edb000 refcount=3 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000020:00000001:1.0:1789036427.606792:0:2393:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036427.606793:0:2393:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2a200. 02000000:00000001:1.0:1789036427.606795:0:2393:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036427.606795:0:2393:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036427.606797:0:2393:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f78700. 00000100:00000001:1.0:1789036427.606798:0:2393:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036427.606800:0:2393:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.606803:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.606805:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.606808:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1789036427.606837:0:2393:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036427.606839:0:2393:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036427.606842:0:2393:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903e210. 00000400:00000200:1.0:1789036427.606846:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d000 00000400:00000010:1.0:1789036427.606848:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d000. 00000100:00000001:1.0:1789036427.606852:0:2393:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036427.606857:0:2393:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036427.606866:0:2393:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.606884:0:2393:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036427.606886:0:2393:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.606889:0:2393:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036427.606891:0:2393:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036427.606893:0:2393:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036427.606896:0:2393:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.606899:0:2393:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036427.606909:0:2393:0:(import.c:2196:obd_at_measure()) add 36 to ffff97d887d65460 time=88 v=36 (36 5 5 5) 00000100:00001000:1.0:1789036427.606913:0:2393:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887d65418 time=16 v=5 (1 1 1 1) 00000100:00000001:1.0:1789036427.606916:0:2393:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036427.606918:0:2393:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036427.606920:0:2393:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1789036427.606923:0:2393:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.606927:0:2393:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1789036427.606929:0:2393:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff97d8b9387800: skip recheck: last_committed 42949695659 00000100:00000001:1.0:1789036427.606931:0:2393:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1789036427.606933:0:2393:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.606938:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.606945:0:2393:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.606948:0:2393:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036427.606950:0:2393:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036427.606953:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036427.606961:0:2393:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f7bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346551808:192.168.204.121@tcp:400:kworker.0 00000400:00000001:1.0:1789036427.606968:0:2393:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036427.606969:0:2393:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036427.606971:0:2393:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903e580. 00000400:00000200:1.0:1789036427.606973:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d4c8 00000400:00000010:1.0:1789036427.606975:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d4c8. 00000100:00000001:1.0:1789036427.606978:0:2393:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036427.606986:0:2393:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036427.606994:0:2393:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.607000:0:2393:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036427.607001:0:2393:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.607003:0:2393:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036427.607005:0:2393:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036427.607006:0:2393:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036427.607007:0:2393:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036427.607009:0:2393:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036427.607012:0:2393:0:(import.c:2196:obd_at_measure()) add 5 to ffff97d8861f3c60 time=88 v=5 (5 5 5 5) 00000100:00001000:1.0:1789036427.607015:0:2393:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d8861f3c18 time=124 v=5 (1 1 0 0) 00000100:00000001:1.0:1789036427.607017:0:2393:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036427.607018:0:2393:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036427.607020:0:2393:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1789036427.607022:0:2393:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.607023:0:2393:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036427.607027:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.607033:0:2393:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036427.607034:0:2393:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036427.607036:0:2393:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036427.607039:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036427.607046:0:2393:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f7a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2393:1875927346551424:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036427.607050:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.607052:0:2393:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036427.607054:0:2393:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f7bb80 x1875927346551808/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036482 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.607060:0:2393:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036427.607068:0:2393:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036427.607069:0:2393:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8a7c60a00. 02000000:00000001:1.0:1789036427.607072:0:2393:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036427.607073:0:2393:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036427.607074:0:2393:0:(genops.c:1222:class_import_put()) import ffff97d887d65000 refcount=3 obd=lustre-OST0000-osc-ffff97d8b9387800 00000020:00000001:1.0:1789036427.607076:0:2393:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036427.607078:0:2393:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2af00. 02000000:00000001:1.0:1789036427.607079:0:2393:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036427.607080:0:2393:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036427.607082:0:2393:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f7bb80. 00000100:00000001:1.0:1789036427.607084:0:2393:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036427.607085:0:2393:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.607087:0:2393:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036427.607090:0:2393:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f7a300 x1875927346551424/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036443 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036427.607096:0:2393:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036427.607097:0:2393:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036427.607099:0:2393:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8a7c61800. 02000000:00000001:1.0:1789036427.607101:0:2393:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036427.607102:0:2393:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036427.607103:0:2393:0:(genops.c:1222:class_import_put()) import ffff97d8861f3800 refcount=3 obd=MGC192.168.204.121@tcp 00000020:00000001:1.0:1789036427.607105:0:2393:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036427.607107:0:2393:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2ab00. 02000000:00000001:1.0:1789036427.607108:0:2393:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036427.607109:0:2393:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036427.607110:0:2393:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f7a300. 00000100:00000001:1.0:1789036427.607113:0:2393:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036427.607114:0:2393:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.607116:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036427.607122:0:2393:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036427.607124:0:2393:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036427.607126:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036427.607128:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036427.845546:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036427.845557:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036427.845565:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036427.845566:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789036427.846107:0:236343:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789036427.846111:0:236343:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789036427.846115:0:236343:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036428.896213:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036428.896220:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036428.896226:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036428.896229:0:274228:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1789036428.896231:0:274228:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1789036428.896237:0:274228:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1789036428.896239:0:274228:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1789036428.896241:0:274228:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036428.896244:0:274228:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1789036428.896246:0:274228:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036428.896248:0:274228:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036428.896250:0:274228:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036428.900286:0:248124:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036428.900293:0:248124:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036428.900298:0:248124:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036429.953960:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036429.953967:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036429.953973:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036429.953974:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036429.954116:0:242419:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036429.954121:0:242419:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036429.954124:0:242419:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036430.975204:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036430.975211:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036430.975216:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036430.975217:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1789036430.977308:0:236342:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1789036430.977313:0:236342:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1789036430.977317:0:236342:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1789036431.422943:0:274228:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712ca20 (cpu 2) has reached its deadline 4307512534, at 4307512576 00000100:00000010:2.0:1789036431.422959:0:274228:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d887b737e0. 00000100:00000040:3.0:1789036431.426948:0:252176:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712cb30 (cpu 3) has reached its deadline 4307512534, at 4307512580 00000100:00000010:3.0:1789036431.426970:0:252176:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d887b738a0. 00010000:00000001:2.0:1789036431.999792:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036431.999801:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036431.999863:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036431.999866:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036432.000044:0:252249:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036432.000048:0:252249:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036432.000051:0:252249:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1789036432.703858:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036432.703869:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036432.703878:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac505880. 00000020:00000040:2.0:1789036432.703885:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d8861f3800 refcount=4 obd=MGC192.168.204.121@tcp 00000100:00000001:2.0:1789036432.703890:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036432.703892:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036432.703897:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036432.703906:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d400. 00000100:00000001:2.0:1789036432.703914:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.703924:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036432.703934:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036432.703938:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.703943:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d317f492] to pc [ptlrpcd_01_01+1] req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.703982:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.703985:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036432.703988:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036432.703991:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac505180. 00000020:00000040:2.0:1789036432.704008:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d887edf000 refcount=6 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000100:00000001:2.0:1789036432.704010:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036432.704012:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036432.704014:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036432.704017:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d500. 00000100:00000001:2.0:1789036432.704020:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704025:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036432.704032:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036432.704034:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704037:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000318f8a63] to pc [ptlrpcd_01_00+0] req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.704045:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704047:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036432.704050:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036432.704053:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac507100. 00000020:00000040:2.0:1789036432.704055:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d887edb000 refcount=4 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000100:00000001:2.0:1789036432.704057:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036432.704058:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036432.704060:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036432.704064:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d900. 00000100:00000001:2.0:1789036432.704066:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704069:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036432.704075:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036432.704076:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704079:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ccc2854] to pc [ptlrpcd_01_01+1] req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.704085:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704087:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036432.704096:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036432.704099:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac505c00. 00000020:00000040:2.0:1789036432.704102:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d887d65000 refcount=4 obd=lustre-OST0000-osc-ffff97d8b9387800 00000100:00000001:2.0:1789036432.704103:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036432.704105:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036432.704107:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036432.704110:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6da00. 00000100:00000001:2.0:1789036432.704112:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704116:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036432.704123:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036432.704124:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704127:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d3c34d63] to pc [ptlrpcd_01_00+0] req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.704133:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036432.704135:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1789036432.704138:0:233053:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1789036432.704142:0:233053:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12851) 00000100:00000001:3.0:1789036432.704284:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.704288:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036432.704290:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1789036432.704297:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.704304:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1789036432.704306:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.704310:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1789036432.704312:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789036432.704314:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789036432.704318:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2396:1875927346551936:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036432.704322:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1789036432.704327:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1789036432.704328:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.704331:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1789036432.704335:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26de00. 02000000:00000001:3.0:1789036432.704337:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789036432.704344:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:3.0:1789036432.704348:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8912068f0. 00000400:00000010:3.0:1789036432.704354:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44f7f8. 00000100:00000200:3.0:1789036432.704361:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346551936, portal 25 00000100:00000001:3.0:1789036432.704364:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1789036432.704366:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592517760 : -114518117033856 : ffff97d8ac505880) 00000100:00000040:3.0:1789036432.704371:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.704378:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789036432.704381:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:3.0:1789036432.704383:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44fa18. 00000100:00000200:3.0:1789036432.704386:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875927346551936, offset 0 00000400:00000200:3.0:1789036432.704392:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:3.0:1789036432.704400:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.704407:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:3.0:1789036432.704411:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1789036432.704423:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:3.0:1789036432.704426:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1789036432.704429:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:3.0:1789036432.704433:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556982:556982:256:4294967295] 192.168.204.121@tcp LPNI seq info [556936:556936:8:4294967295] 00000400:00000200:3.0:1789036432.704438:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.704457:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:3.0:1789036432.704464:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:3.0:1789036432.704478:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d88536e500. 00000800:00000200:3.0:1789036432.704485:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:3.0:1789036432.704492:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:3.0:1789036432.704496:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d88536e500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789036432.704511:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789036432.704515:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:3.0:1789036432.704518:0:2396:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 2 is 4307517655 00000100:00000010:3.0:1789036432.704522:0:2396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8b9d5b3c0. 00000100:00000001:3.0:1789036432.704528:0:2396:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1789036432.704533:0:2396:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 3 is 4307517655 00000100:00000010:3.0:1789036432.704641:0:2396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8b9d5b7e0. 00000100:00000001:3.0:1789036432.704645:0:2396:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1789036432.704648:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.704651:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.704653:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1789036432.704657:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.704664:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1789036432.704665:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.704667:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1789036432.704669:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789036432.704670:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789036432.704673:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac507100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552192:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036432.704683:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1789036432.704686:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1789036432.704687:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.704689:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1789036432.704692:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26c200. 02000000:00000001:3.0:1789036432.704694:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789036432.704698:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:3.0:1789036432.704701:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8912066e0. 00000400:00000010:3.0:1789036432.704705:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44f6e8. 00000100:00000200:3.0:1789036432.704708:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552192, portal 10 00000100:00000001:3.0:1789036432.704710:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1789036432.704711:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592524032 : -114518117027584 : ffff97d8ac507100) 00000100:00000040:3.0:1789036432.704718:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.704724:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789036432.704726:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:3.0:1789036432.704729:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44fee0. 00000100:00000200:3.0:1789036432.704731:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346552192, offset 0 00000400:00000200:3.0:1789036432.704735:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:3.0:1789036432.704739:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.704742:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:3.0:1789036432.704744:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1789036432.704747:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:3.0:1789036432.704749:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1789036432.704750:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:3.0:1789036432.704753:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556983:556983:255:4294967295] 192.168.204.121@tcp LPNI seq info [556937:556937:7:4294967295] 00000400:00000200:3.0:1789036432.704757:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.704762:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:3.0:1789036432.704767:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:3.0:1789036432.704770:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d88536e100. 00000800:00000200:3.0:1789036432.704774:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:3.0:1789036432.704782:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:3.0:1789036432.704785:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d88536e100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789036432.704787:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.704791:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: using a timeout of 19615 usecs (20 jiffies) 00000400:00000200:3.0:1789036432.704794:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:3.0:1789036432.704796:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307513877 00000100:00000001:3.0:1789036432.704798:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307517655 00000100:00000001:3.0:1789036432.704800:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307513877 00000100:00000001:3.0:1789036432.704801:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307517655 00000100:00000001:3.0:1789036432.704803:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.704805:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.704807:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.704809:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.705091:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789036432.705094:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1789036432.705096:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.705098:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036432.705100:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789036432.705102:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:3.0:1789036432.705104:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.705106:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789036432.705107:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.705109:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.705109:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036432.705110:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1789036432.705116:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.705122:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1789036432.705124:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.705126:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1789036432.705128:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789036432.705130:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789036432.705133:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552064:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036432.705142:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1789036432.705146:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1789036432.705148:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.705149:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1789036432.705152:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26d600. 02000000:00000001:3.0:1789036432.705154:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789036432.705158:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:3.0:1789036432.705161:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d891206898. 00000400:00000010:3.0:1789036432.705164:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44f550. 00000100:00000200:3.0:1789036432.705167:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552064, portal 10 00000100:00000001:3.0:1789036432.705168:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1789036432.705170:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592515968 : -114518117035648 : ffff97d8ac505180) 00000100:00000040:3.0:1789036432.705175:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1789036432.706037:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036432.706043:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d88536e500. 00000400:00000200:2.0:1789036432.706050:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.706056:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036432.706061:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44fa18 00000400:00000010:2.0:1789036432.706064:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44fa18. 00000100:00000001:2.0:1789036432.706069:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036432.706073:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.706082:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036432.706086:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.706104:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036432.706106:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789036432.706168:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036432.706170:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d88536e100. 00000400:00000200:2.0:1789036432.706173:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1789036432.706346:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789036432.706350:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:3.0:1789036432.706353:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44fcc0. 00000100:00000200:3.0:1789036432.706357:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346552064, offset 0 00000400:00000200:3.0:1789036432.706362:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:3.0:1789036432.706368:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.706373:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:3.0:1789036432.706376:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1789036432.706379:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:3.0:1789036432.706382:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1789036432.706385:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:3.0:1789036432.706387:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556984:556984:255:4294967295] 192.168.204.121@tcp LPNI seq info [556938:556938:7:4294967295] 00000400:00000200:3.0:1789036432.706392:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.706401:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:3.0:1789036432.706406:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:3.0:1789036432.706410:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d88536ef00. 00000800:00000200:3.0:1789036432.706414:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:3.0:1789036432.706425:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:3.0:1789036432.706427:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d88536ef00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789036432.706431:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.706434:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: using a timeout of 25706 usecs (26 jiffies) 00000400:00000200:3.0:1789036432.706438:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:3.0:1789036432.706440:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307513885 00000100:00000001:3.0:1789036432.706443:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307517655 00000100:00000001:3.0:1789036432.706445:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307513885 00000400:00000200:2.0:1789036432.707722:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036432.707729:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44fee0 00000400:00000010:2.0:1789036432.707732:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44fee0. 00000100:00000001:2.0:1789036432.707736:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036432.707742:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.707750:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036432.707753:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.707759:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036432.707760:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789036432.707831:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036432.707834:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d88536ef00. 00000400:00000200:2.0:1789036432.707837:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.707841:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036432.707844:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44fcc0 00000400:00000010:2.0:1789036432.707846:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44fcc0. 00000100:00000001:2.0:1789036432.707848:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036432.707851:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.707858:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036432.707860:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.707867:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036432.707868:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1789036432.707895:0:2395:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036432.707906:0:2395:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.708108:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307517655 00000100:00000001:3.0:1789036432.708111:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.708114:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.708118:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1789036432.708123:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.708130:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1789036432.708132:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.708135:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1789036432.708138:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789036432.708140:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789036432.708144:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac505c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552320:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036432.708148:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1789036432.708152:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1789036432.708155:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.708157:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1789036432.708161:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26d400. 02000000:00000001:3.0:1789036432.708163:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789036432.708168:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:3.0:1789036432.708171:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d891206580. 00000400:00000010:3.0:1789036432.709433:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44f3b8. 00000100:00000200:3.0:1789036432.709444:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552320, portal 4 00000100:00000001:3.0:1789036432.709456:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1789036432.709476:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592518656 : -114518117032960 : ffff97d8ac505c00) 00000100:00000040:3.0:1789036432.709487:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.709494:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789036432.709497:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:3.0:1789036432.709501:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac44f440. 00000100:00000200:3.0:1789036432.709504:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875927346552320, offset 0 00000400:00000200:3.0:1789036432.709509:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:3.0:1789036432.709517:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.709523:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:3.0:1789036432.709527:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1789036432.709530:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:3.0:1789036432.709533:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1789036432.710794:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:3.0:1789036432.710798:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556985:556985:256:4294967295] 192.168.204.121@tcp LPNI seq info [556939:556939:8:4294967295] 00000400:00000200:3.0:1789036432.710804:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036432.710813:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:3.0:1789036432.710834:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:3.0:1789036432.710840:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d88536ea00. 00000800:00000200:3.0:1789036432.710846:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:3.0:1789036432.710853:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:3.0:1789036432.710857:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d88536ea00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789036432.710879:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.710885:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: using a timeout of 102788 usecs (103 jiffies) 00000400:00000200:3.0:1789036432.710888:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:3.0:1789036432.710891:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307513967 00000100:00000001:3.0:1789036432.710894:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307517655 00000100:00000001:3.0:1789036432.710896:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307513967 00000100:00000001:3.0:1789036432.710898:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307517655 00000100:00000001:3.0:1789036432.710909:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.710912:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.710916:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.710920:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789036432.718141:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036432.718153:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d88536ea00. 00000400:00000200:2.0:1789036432.718163:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.718184:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036432.718193:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44f440 00000400:00000010:2.0:1789036432.718196:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44f440. 00000100:00000001:2.0:1789036432.718202:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036432.718210:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.718222:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036432.718225:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.718232:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036432.718234:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1789036432.719454:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719466:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036432.719478:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719482:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.719490:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719496:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036432.719503:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719507:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719512:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719516:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1d9d80 00000400:00000200:2.0:1789036432.719521:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719526:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x539275 [1] + 224 00000800:00000001:2.0:1789036432.719541:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719546:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.719549:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.719553:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036432.719559:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036432.719564:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036432.719590:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.719609:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036432.719619:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719623:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036432.719625:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719627:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.719630:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719634:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036432.719638:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719641:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719644:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719647:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 25 MB=0x6aa255f1d9c80 00000400:00000200:2.0:1789036432.719651:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719656:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.121@tcp of length 224/224 into md 0x539265 [1] + 224 00000800:00000001:2.0:1789036432.719659:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719662:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.719664:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.719667:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036432.719670:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036432.719674:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036432.719683:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.719691:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036432.719695:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719698:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036432.719699:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719701:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.719705:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719715:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036432.719719:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719722:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719725:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719728:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1d9d00 00000400:00000200:2.0:1789036432.719731:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.719734:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x539285 [1] + 224 00000800:00000001:2.0:1789036432.719738:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719745:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.719747:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.719751:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036432.719754:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036432.719758:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036432.719767:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.719774:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036432.719779:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.719782:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.719838:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.719842:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789036432.719848:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036432.719850:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036432.719852:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d891206898. 00000400:00000200:3.0:1789036432.719856:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44f550 00000400:00000010:3.0:1789036432.719858:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44f550. 00000100:00000001:3.0:1789036432.719861:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036432.719865:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036432.719872:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.719878:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036432.719879:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.719881:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036432.719884:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036432.719887:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036432.719889:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.719892:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036432.719901:0:2396:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edf460 time=93 v=60 (60 61 61 62) 00000100:00001000:3.0:1789036432.719904:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edf418 time=129 v=5 (1 1 0 0) 00000100:00000001:3.0:1789036432.719907:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036432.719908:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036432.719911:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789036432.719914:0:2396:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.719917:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789036432.719919:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff97d8b9387800: skip recheck: last_committed 30064771460 00000100:00000001:3.0:1789036432.719921:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789036432.719922:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.719927:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.719943:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.719946:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036432.719948:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036432.719952:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036432.719959:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552064:192.168.204.121@tcp:400:kworker.0 00000400:00000001:3.0:1789036432.719967:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036432.719969:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036432.719970:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8912068f0. 00000400:00000200:3.0:1789036432.719972:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44f7f8 00000400:00000010:3.0:1789036432.719974:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44f7f8. 00000100:00000001:3.0:1789036432.719976:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036432.719979:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036432.719986:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.719990:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036432.719992:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.719993:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036432.719995:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036432.719996:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036432.719997:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.719999:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036432.720002:0:2396:0:(import.c:2196:obd_at_measure()) add 5 to ffff97d8861f3c60 time=93 v=5 (5 5 5 5) 00000100:00001000:3.0:1789036432.720004:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d8861f3c18 time=129 v=5 (1 1 0 0) 00000100:00000001:3.0:1789036432.720006:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036432.720007:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036432.720009:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789036432.720011:0:2396:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.720013:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.720016:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.720028:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.720030:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036432.720031:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036432.720035:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036432.720042:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2396:1875927346551936:192.168.204.121@tcp:400:kworker.0 00000400:00000001:3.0:1789036432.720048:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036432.720049:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036432.720051:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8912066e0. 00000400:00000200:3.0:1789036432.720053:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44f6e8 00000400:00000010:3.0:1789036432.720055:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44f6e8. 00000100:00000001:3.0:1789036432.720057:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036432.720061:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036432.720069:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.720075:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036432.720076:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.720078:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036432.720079:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036432.720081:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036432.720082:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.720084:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036432.720089:0:2396:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edb460 time=93 v=60 (60 61 61 62) 00000100:00001000:3.0:1789036432.720092:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edb418 time=106 v=5 (1 1 0 0) 00000100:00000001:3.0:1789036432.720095:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036432.720096:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036432.720098:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1789036432.720101:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789036432.720118:0:2396:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.720120:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789036432.720122:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff97d8b9387800: skip recheck: last_committed 21474836540 00000100:00000001:3.0:1789036432.720124:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789036432.720125:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.720129:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.720136:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.720138:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036432.720139:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036432.720143:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036432.720151:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac507100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552192:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036432.720155:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.720157:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036432.720160:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac505180 x1875927346552064/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.720166:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036432.720168:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036432.720172:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26d600. 02000000:00000001:3.0:1789036432.720174:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036432.720190:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036432.720192:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887edf000 refcount=5 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000020:00000001:3.0:1789036432.720194:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036432.720196:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d500. 02000000:00000001:3.0:1789036432.720200:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036432.720202:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036432.720203:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac505180. 00000100:00000001:3.0:1789036432.720206:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036432.720207:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.720214:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036432.720218:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac505880 x1875927346551936/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036448 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.720223:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036432.720225:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036432.720226:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26de00. 02000000:00000001:3.0:1789036432.720228:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036432.720229:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036432.720230:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d8861f3800 refcount=3 obd=MGC192.168.204.121@tcp 00000020:00000001:3.0:1789036432.720232:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036432.720233:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d400. 02000000:00000001:3.0:1789036432.720235:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036432.720236:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036432.720238:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac505880. 00000100:00000001:3.0:1789036432.720240:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036432.720241:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.720243:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036432.720245:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac507100 x1875927346552192/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036517 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.720251:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036432.720252:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036432.720254:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26c200. 02000000:00000001:3.0:1789036432.720256:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036432.720256:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036432.720258:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887edb000 refcount=3 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000020:00000001:3.0:1789036432.720259:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036432.720260:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d900. 02000000:00000001:3.0:1789036432.720262:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036432.720263:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036432.720264:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac507100. 00000100:00000001:3.0:1789036432.720266:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036432.720267:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.720270:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.720272:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.720273:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036432.720275:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.720277:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789036432.723364:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.723382:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036432.723385:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.723388:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.723397:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.723404:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036432.723414:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.723418:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.723422:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.723426:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 4 MB=0x6aa255f1d9e00 00000400:00000200:2.0:1789036432.723431:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036432.723436:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.121@tcp of length 224/224 into md 0x539295 [1] + 224 00000800:00000001:2.0:1789036432.723442:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.723485:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036432.723489:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036432.723494:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036432.723501:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036432.723509:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036432.723519:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036432.723548:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036432.723556:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036432.723560:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.723716:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.723721:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789036432.723727:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036432.723729:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036432.723732:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d891206580. 00000400:00000200:3.0:1789036432.723736:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac44f3b8 00000400:00000010:3.0:1789036432.723738:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac44f3b8. 00000100:00000001:3.0:1789036432.723741:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036432.723746:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036432.723755:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.723761:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036432.723763:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.723765:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036432.723768:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036432.723770:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036432.723772:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036432.723775:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036432.723784:0:2396:0:(import.c:2196:obd_at_measure()) add 36 to ffff97d887d65460 time=93 v=36 (36 5 5 5) 00000100:00001000:3.0:1789036432.723788:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887d65418 time=21 v=5 (1 1 1 1) 00000100:00000001:3.0:1789036432.723791:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036432.723792:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036432.723795:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789036432.723798:0:2396:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.723802:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789036432.723804:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff97d8b9387800: skip recheck: last_committed 42949695659 00000100:00000001:3.0:1789036432.723805:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789036432.723807:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036432.723812:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.723847:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036432.723849:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036432.723851:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036432.723855:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036432.723872:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac505c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552320:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036432.723877:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.723879:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036432.723882:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac505c00 x1875927346552320/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036487 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036432.723888:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036432.723890:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036432.723894:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26d400. 02000000:00000001:3.0:1789036432.723897:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036432.723899:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036432.723901:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887d65000 refcount=3 obd=lustre-OST0000-osc-ffff97d8b9387800 00000020:00000001:3.0:1789036432.723903:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036432.723905:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6da00. 02000000:00000001:3.0:1789036432.723909:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036432.723910:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036432.723912:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac505c00. 00000100:00000001:3.0:1789036432.723915:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036432.723917:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.723919:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036432.723927:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789036432.723929:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036432.723931:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036432.723933:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036433.023220:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036433.023229:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036433.023236:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036433.023238:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789036433.023266:0:236343:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789036433.023270:0:236343:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789036433.023274:0:236343:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036434.047294:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036434.047302:0:274228:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1789036434.047305:0:274228:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1789036434.047308:0:274228:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036434.047313:0:274228:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1789036434.047315:0:274228:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036434.047330:0:274228:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036434.047332:0:274228:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036434.047336:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036434.047337:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789036434.047714:0:248124:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789036434.047719:0:248124:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789036434.047722:0:248124:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036435.074469:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036435.074477:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036435.074483:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036435.074484:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036435.074894:0:242419:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036435.074898:0:242419:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036435.074902:0:242419:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036436.101115:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036436.101130:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036436.101136:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036436.101143:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036436.101413:0:236342:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036436.101416:0:236342:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036436.101419:0:236342:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:2.0:1789036436.273447:0:278787:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036436.543989:0:252176:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712cb30 (cpu 3) has reached its deadline 4307517655, at 4307517697 00000100:00000010:3.0:1789036436.544006:0:252176:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8b9d5b7e0. 00000100:00000040:2.0:1789036436.547265:0:274228:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712ca20 (cpu 2) has reached its deadline 4307517655, at 4307517698 00000100:00000010:2.0:1789036436.547280:0:274228:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8b9d5b3c0. 00010000:00000001:2.0:1789036437.119602:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036437.119611:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036437.119706:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036437.119708:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036437.130001:0:252249:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036437.130008:0:252249:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036437.130013:0:252249:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1789036437.823447:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789036437.823461:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789036437.823469:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8a7d95500. 00000020:00000040:3.0:1789036437.823475:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d8861f3800 refcount=4 obd=MGC192.168.204.121@tcp 00000100:00000001:3.0:1789036437.823479:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789036437.823483:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789036437.823495:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789036437.823503:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d88536e800. 00000100:00000001:3.0:1789036437.823511:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823525:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789036437.823536:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789036437.823541:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823547:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094e79b3a] to pc [ptlrpcd_01_01+1] req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.823577:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823581:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789036437.823584:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789036437.823587:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8a7d97100. 00000020:00000040:3.0:1789036437.823589:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d887edf000 refcount=6 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000100:00000001:3.0:1789036437.823591:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789036437.823592:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789036437.823594:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789036437.823597:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d88536e200. 00000100:00000001:3.0:1789036437.823600:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823604:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789036437.823610:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789036437.823612:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823616:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003252b956] to pc [ptlrpcd_01_00+0] req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.823624:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823626:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789036437.823629:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789036437.823632:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8a7d97480. 00000020:00000040:3.0:1789036437.823634:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d887edb000 refcount=4 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000100:00000001:3.0:1789036437.823643:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789036437.823644:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789036437.823646:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789036437.823649:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d88536e900. 00000100:00000001:3.0:1789036437.823652:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823659:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789036437.823667:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789036437.823669:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823673:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e45e5059] to pc [ptlrpcd_01_01+1] req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.823680:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823682:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789036437.823685:0:233053:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789036437.823687:0:233053:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8a7d95880. 00000020:00000040:3.0:1789036437.823689:0:233053:0:(genops.c:1209:class_import_get()) import ffff97d887d65000 refcount=4 obd=lustre-OST0000-osc-ffff97d8b9387800 00000100:00000001:3.0:1789036437.823691:0:233053:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789036437.823692:0:233053:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789036437.823694:0:233053:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789036437.823697:0:233053:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d88536ec00. 00000100:00000001:3.0:1789036437.823699:0:233053:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823703:0:233053:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789036437.823709:0:233053:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789036437.823711:0:233053:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823727:0:233053:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083ff8e2a] to pc [ptlrpcd_01_00+0] req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.823734:0:233053:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036437.823736:0:233053:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1789036437.823740:0:233053:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1789036437.823750:0:233053:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12856) 00000100:00000001:2.0:1789036437.825147:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036437.825153:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036437.825155:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036437.825166:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.825176:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036437.825179:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.825183:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036437.825186:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036437.825189:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036437.825193:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8a7d95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2396:1875927346552448:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036437.825197:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036437.825203:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036437.825206:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.825208:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036437.825213:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d887834e00. 02000000:00000001:2.0:1789036437.825216:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036437.825225:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036437.825230:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb1bb0. 00000400:00000010:2.0:1789036437.825237:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739bcc0. 00000100:00000200:2.0:1789036437.825244:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552448, portal 25 00000100:00000001:2.0:1789036437.825247:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036437.825249:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555517609216 : -114518191942400 : ffff97d8a7d95500) 00000100:00000040:2.0:1789036437.825254:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.825261:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036437.825269:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036437.825272:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b088. 00000100:00000200:2.0:1789036437.825275:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875927346552448, offset 0 00000400:00000200:2.0:1789036437.825280:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036437.825288:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.825295:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036437.825300:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036437.825304:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036437.825308:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036437.825311:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036437.825315:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556986:556986:256:4294967295] 192.168.204.121@tcp LPNI seq info [556940:556940:8:4294967295] 00000400:00000200:2.0:1789036437.825319:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.825328:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036437.825334:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036437.825340:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6df00. 00000800:00000200:2.0:1789036437.825346:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036437.825364:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036437.825369:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6df00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036437.825390:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036437.825395:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036437.825398:0:2396:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 2 is 4307522776 00000100:00000010:2.0:1789036437.825402:0:2396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8ac080240. 00000100:00000001:2.0:1789036437.825409:0:2396:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1789036437.825415:0:2396:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 3 is 4307522776 00000100:00000010:2.0:1789036437.825418:0:2396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8ac0807e0. 00000100:00000001:2.0:1789036437.825422:0:2396:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1789036437.825425:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.825427:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.825430:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036437.825435:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.825452:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036437.825454:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.825456:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036437.825458:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036437.825460:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036437.825463:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8a7d97480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552704:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036437.825467:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036437.825470:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036437.825472:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.825473:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036437.825477:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d887835600. 02000000:00000001:2.0:1789036437.825479:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036437.825483:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036437.825486:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb1160. 00000400:00000010:2.0:1789036437.825489:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b4c8. 00000100:00000200:2.0:1789036437.825492:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552704, portal 10 00000100:00000001:2.0:1789036437.825494:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036437.825496:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555517617280 : -114518191934336 : ffff97d8a7d97480) 00000100:00000040:2.0:1789036437.825503:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.825510:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036437.825512:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036437.825514:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b660. 00000100:00000200:2.0:1789036437.825517:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346552704, offset 0 00000400:00000200:2.0:1789036437.825521:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036437.825526:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.825529:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036437.825532:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036437.825535:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036437.825537:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036437.825539:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036437.825550:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556987:556987:255:4294967295] 192.168.204.121@tcp LPNI seq info [556941:556941:7:4294967295] 00000400:00000200:2.0:1789036437.825554:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.825560:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036437.825565:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036437.825569:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6d100. 00000800:00000200:2.0:1789036437.825572:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036437.825577:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036437.825579:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6d100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036437.825582:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.825586:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: using a timeout of 19615 usecs (20 jiffies) 00000400:00000200:2.0:1789036437.825589:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036437.825591:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307518998 00000100:00000001:2.0:1789036437.825594:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307522776 00000100:00000001:2.0:1789036437.825595:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307518998 00000100:00000001:2.0:1789036437.825597:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307522776 00000100:00000001:2.0:1789036437.825600:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.825601:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.825604:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.825606:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.825614:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789036437.825616:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789036437.825619:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036437.825620:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036437.825621:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036437.825625:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:2.0:1789036437.825626:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.825628:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789036437.825629:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.825631:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036437.825632:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036437.825634:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036437.825638:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.825651:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036437.825653:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.825656:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036437.825658:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036437.825660:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036437.825663:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8a7d97100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552576:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036437.825666:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036437.825669:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036437.825671:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.825673:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036437.825676:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d887834600. 02000000:00000001:2.0:1789036437.825678:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036437.825681:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036437.825684:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb1528. 00000400:00000010:2.0:1789036437.825688:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b770. 00000100:00000200:2.0:1789036437.825691:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552576, portal 10 00000100:00000001:2.0:1789036437.825693:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036437.825694:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555517616384 : -114518191935232 : ffff97d8a7d97100) 00000100:00000040:2.0:1789036437.825699:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.825706:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036437.825708:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036437.825711:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739bc38. 00000100:00000200:2.0:1789036437.825713:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346552576, offset 0 00000400:00000200:2.0:1789036437.825793:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036437.825797:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.825801:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036437.825803:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036437.825805:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036437.825807:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036437.825809:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036437.825847:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556988:556988:254:4294967295] 192.168.204.121@tcp LPNI seq info [556942:556942:6:4294967295] 00000400:00000200:2.0:1789036437.825851:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.825929:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036437.825935:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036437.825939:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6dc00. 00000800:00000200:3.0:1789036437.826089:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036437.826094:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6df00. 00000400:00000200:3.0:1789036437.826101:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.826106:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036437.826111:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b088 00000400:00000010:3.0:1789036437.826114:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b088. 00000100:00000001:3.0:1789036437.826117:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036437.826122:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.826131:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036437.826135:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.826142:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036437.826143:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036437.826217:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036437.826219:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6d100. 00000400:00000200:3.0:1789036437.826222:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.826225:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036437.826227:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b660 00000400:00000010:3.0:1789036437.826229:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b660. 00000100:00000001:3.0:1789036437.826231:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036437.826234:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.826240:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036437.826243:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.826248:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036437.826249:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1789036437.826278:0:2395:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036437.826280:0:2395:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789036437.826677:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036437.826686:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036437.826689:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6dc00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036437.826701:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.826705:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: using a timeout of 25703 usecs (26 jiffies) 00000400:00000200:2.0:1789036437.826710:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036437.826712:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307519005 00000100:00000001:2.0:1789036437.827270:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307522776 00000100:00000001:2.0:1789036437.827274:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307519006 00000100:00000001:2.0:1789036437.827277:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307522776 00000100:00000001:2.0:1789036437.827279:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.827282:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.827286:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036437.827292:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.827300:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036437.827302:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.827305:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036437.827308:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036437.827310:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036437.827313:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8a7d95880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552832:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036437.827317:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036437.827321:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036437.827323:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.827338:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036437.827342:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d887834800. 02000000:00000001:2.0:1789036437.827344:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036437.827360:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036437.827364:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb10b0. 00000400:00000010:2.0:1789036437.827369:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739be58. 00000100:00000200:2.0:1789036437.827374:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552832, portal 4 00000100:00000001:2.0:1789036437.827377:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036437.827378:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555517610112 : -114518191941504 : ffff97d8a7d95880) 00000100:00000040:2.0:1789036437.827385:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.827392:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036437.827394:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036437.827397:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b908. 00000100:00000200:2.0:1789036437.827399:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875927346552832, offset 0 00000400:00000200:2.0:1789036437.827404:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036437.827409:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.827415:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036437.827419:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036437.827421:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036437.827424:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036437.827427:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036437.827430:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556989:556989:255:4294967295] 192.168.204.121@tcp LPNI seq info [556943:556943:7:4294967295] 00000400:00000200:2.0:1789036437.827435:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036437.827442:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036437.827447:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036437.827451:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6d300. 00000800:00000200:2.0:1789036437.827456:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036437.827460:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036437.827463:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6d300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036437.827466:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.827474:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: using a timeout of 102726 usecs (103 jiffies) 00000400:00000200:2.0:1789036437.827478:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036437.827480:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307519083 00000100:00000001:2.0:1789036437.827482:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307522776 00000100:00000001:2.0:1789036437.827484:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307519083 00000100:00000001:2.0:1789036437.827486:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307522776 00000100:00000001:2.0:1789036437.827488:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.827491:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.827494:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.827498:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1789036437.827909:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036437.827918:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6dc00. 00000400:00000200:3.0:1789036437.827923:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.827927:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036437.827931:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739bc38 00000400:00000010:3.0:1789036437.827933:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739bc38. 00000100:00000001:3.0:1789036437.827936:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036437.827941:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.827956:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036437.827960:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.827966:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036437.827967:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036437.828417:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036437.828421:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6d300. 00000400:00000200:3.0:1789036437.828424:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.828429:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036437.828433:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b908 00000400:00000010:3.0:1789036437.828435:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b908. 00000100:00000001:3.0:1789036437.828438:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036437.828442:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.828450:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036437.828453:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.828459:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036437.828461:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:3.0:1789036437.840486:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840503:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036437.840507:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840510:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840518:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840526:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036437.840545:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840550:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840556:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840560:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1d9f80 00000400:00000200:3.0:1789036437.840566:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840571:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392b5 [1] + 224 00000800:00000001:3.0:1789036437.840578:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840583:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840586:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.840590:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036437.840597:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036437.840606:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036437.840618:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.840639:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036437.840648:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840652:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036437.840653:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840656:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840659:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840663:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036437.840667:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840670:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840672:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840675:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1d9f00 00000400:00000200:3.0:1789036437.840678:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840681:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392c5 [1] + 224 00000800:00000001:3.0:1789036437.840685:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840687:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840689:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.840698:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036437.840701:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036437.840706:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036437.840726:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.840733:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036437.840738:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840742:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036437.840744:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840746:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840749:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840752:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036437.840756:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840758:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840761:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840764:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 4 MB=0x6aa255f1da000 00000400:00000200:3.0:1789036437.840767:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840772:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392d5 [1] + 224 00000800:00000001:3.0:1789036437.840775:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840778:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840780:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.840783:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036437.840785:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036437.840790:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036437.840798:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.840805:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036437.840808:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840860:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036437.840862:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840865:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840868:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840872:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036437.840876:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840879:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840882:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840884:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 25 MB=0x6aa255f1d9e80 00000400:00000200:3.0:1789036437.840888:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036437.840892:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392a5 [1] + 224 00000800:00000001:3.0:1789036437.840896:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840909:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036437.840911:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036437.840914:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036437.840917:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036437.840922:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036437.840930:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036437.840938:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036437.840941:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036437.840954:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.845441:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036437.845449:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1789036437.845458:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789036437.845461:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789036437.845465:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb1528. 00000400:00000200:2.0:1789036437.845472:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b770 00000400:00000010:2.0:1789036437.845474:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b770. 00000100:00000001:2.0:1789036437.845479:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036437.845490:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789036437.845501:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845510:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789036437.845511:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.845514:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789036437.845517:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789036437.845520:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789036437.845522:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.845525:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789036437.845536:0:2396:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edf460 time=98 v=60 (60 61 61 62) 00000100:00001000:2.0:1789036437.845539:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edf418 time=134 v=5 (1 1 0 0) 00000100:00000001:2.0:1789036437.845542:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789036437.845543:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036437.845555:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789036437.845560:0:2396:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.845564:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789036437.845566:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff97d8b9387800: skip recheck: last_committed 30064771460 00000100:00000001:2.0:1789036437.845568:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789036437.845570:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.845575:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845583:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.845587:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789036437.845589:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789036437.845593:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789036437.845600:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8a7d97100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552576:192.168.204.121@tcp:400:kworker.0 00000400:00000001:2.0:1789036437.845607:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789036437.845608:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789036437.845611:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb10b0. 00000400:00000200:2.0:1789036437.845614:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739be58 00000400:00000010:2.0:1789036437.845615:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739be58. 00000100:00000001:2.0:1789036437.845617:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036437.845621:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789036437.845629:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845635:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789036437.845636:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.845638:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789036437.845640:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789036437.845641:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789036437.845643:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.845644:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789036437.845649:0:2396:0:(import.c:2196:obd_at_measure()) add 36 to ffff97d887d65460 time=98 v=36 (36 5 5 5) 00000100:00001000:2.0:1789036437.845658:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887d65418 time=26 v=5 (1 1 1 1) 00000100:00000001:2.0:1789036437.845660:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789036437.845661:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036437.845663:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789036437.845665:0:2396:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.845667:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789036437.845668:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff97d8b9387800: skip recheck: last_committed 42949695659 00000100:00000001:2.0:1789036437.845670:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789036437.845672:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.845676:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845683:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.845684:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789036437.845686:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789036437.845689:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789036437.845697:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8a7d95880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552832:192.168.204.121@tcp:400:kworker.0 00000400:00000001:2.0:1789036437.845703:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789036437.845704:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789036437.845706:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb1bb0. 00000400:00000200:2.0:1789036437.845708:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739bcc0 00000400:00000010:2.0:1789036437.845710:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739bcc0. 00000100:00000001:2.0:1789036437.845723:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036437.845728:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789036437.845736:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845742:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789036437.845744:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.845745:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789036437.845747:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789036437.845748:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789036437.845756:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.845758:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789036437.845762:0:2396:0:(import.c:2196:obd_at_measure()) add 5 to ffff97d8861f3c60 time=98 v=5 (5 5 5 5) 00000100:00001000:2.0:1789036437.845764:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d8861f3c18 time=134 v=5 (1 1 0 0) 00000100:00000001:2.0:1789036437.845767:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789036437.845768:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036437.845769:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789036437.845771:0:2396:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.845773:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.845777:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845784:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.845786:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789036437.845787:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789036437.845790:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789036437.845797:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8a7d95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2396:1875927346552448:192.168.204.121@tcp:400:kworker.0 00000400:00000001:2.0:1789036437.845803:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789036437.845805:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789036437.845807:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb1160. 00000400:00000200:2.0:1789036437.845809:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b4c8 00000400:00000010:2.0:1789036437.845810:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b4c8. 00000100:00000001:2.0:1789036437.845812:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036437.845853:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789036437.845869:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845875:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789036437.845877:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.845879:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789036437.845880:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789036437.845881:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789036437.845888:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036437.845890:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789036437.845894:0:2396:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edb460 time=98 v=60 (60 61 61 62) 00000100:00001000:2.0:1789036437.845897:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edb418 time=111 v=5 (1 1 0 0) 00000100:00000001:2.0:1789036437.845900:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789036437.845901:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036437.845903:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1789036437.845907:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1789036437.845915:0:2396:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.845917:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789036437.845918:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff97d8b9387800: skip recheck: last_committed 21474836540 00000100:00000001:2.0:1789036437.845920:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789036437.845921:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036437.845924:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.845929:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036437.845931:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789036437.845932:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789036437.845935:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789036437.846366:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8a7d97480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346552704:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036437.846371:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.846374:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036437.846378:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8a7d97100 x1875927346552576/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.846385:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789036437.846387:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789036437.846391:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d887834600. 02000000:00000001:2.0:1789036437.846395:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789036437.846398:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789036437.846406:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887edf000 refcount=5 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000020:00000001:2.0:1789036437.846408:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789036437.846412:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d88536e200. 02000000:00000001:2.0:1789036437.846417:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789036437.846419:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789036437.846420:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8a7d97100. 00000100:00000001:2.0:1789036437.846424:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789036437.846425:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.846427:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036437.846431:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8a7d95880 x1875927346552832/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036492 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.846438:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789036437.846439:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789036437.846441:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d887834800. 02000000:00000001:2.0:1789036437.846443:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789036437.846444:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789036437.846445:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887d65000 refcount=3 obd=lustre-OST0000-osc-ffff97d8b9387800 00000020:00000001:2.0:1789036437.846447:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789036437.846449:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d88536ec00. 02000000:00000001:2.0:1789036437.846451:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789036437.846452:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789036437.846453:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8a7d95880. 00000100:00000001:2.0:1789036437.846456:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789036437.846457:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.846459:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036437.846462:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8a7d95500 x1875927346552448/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036453 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.846468:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789036437.846469:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789036437.846471:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d887834e00. 02000000:00000001:2.0:1789036437.846473:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789036437.846474:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789036437.846476:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d8861f3800 refcount=3 obd=MGC192.168.204.121@tcp 00000020:00000001:2.0:1789036437.846477:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789036437.846478:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d88536e800. 02000000:00000001:2.0:1789036437.846480:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789036437.846488:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789036437.846489:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8a7d95500. 00000100:00000001:2.0:1789036437.846491:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789036437.846493:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.846494:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036437.846498:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8a7d97480 x1875927346552704/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036522 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036437.846504:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789036437.846505:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789036437.846507:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d887835600. 02000000:00000001:2.0:1789036437.846509:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789036437.846510:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789036437.846511:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887edb000 refcount=3 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000020:00000001:2.0:1789036437.846513:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789036437.846515:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d88536e900. 02000000:00000001:2.0:1789036437.846517:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789036437.846518:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789036437.846519:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8a7d97480. 00000100:00000001:2.0:1789036437.846521:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789036437.846522:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.846524:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036437.846532:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789036437.846534:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036437.846536:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036437.846539:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036438.143986:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036438.143994:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036438.143999:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036438.144001:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789036438.144131:0:236343:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789036438.144134:0:236343:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789036438.144137:0:236343:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036439.166981:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036439.166988:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036439.166994:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036439.166998:0:274228:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1789036439.167000:0:274228:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1789036439.167025:0:274228:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1789036439.167027:0:274228:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1789036439.167029:0:274228:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036439.167032:0:274228:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1789036439.167034:0:274228:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036439.167036:0:274228:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036439.167039:0:274228:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1789036439.167632:0:248124:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1789036439.167641:0:248124:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1789036439.167644:0:248124:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036440.193183:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036440.193191:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036440.193197:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036440.193198:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036440.195447:0:242419:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036440.195489:0:242419:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036440.195495:0:242419:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036441.217469:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036441.217476:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036441.217482:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036441.217483:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036441.217940:0:236342:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036441.217944:0:236342:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036441.217947:0:236342:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1789036441.662971:0:252176:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712cb30 (cpu 3) has reached its deadline 4307522776, at 4307522816 00000100:00000010:3.0:1789036441.662986:0:252176:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8ac0807e0. 00000100:00000040:2.0:1789036441.663235:0:274228:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712ca20 (cpu 2) has reached its deadline 4307522776, at 4307522816 00000100:00000010:2.0:1789036441.663242:0:274228:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8ac080240. 00010000:00000001:2.0:1789036442.239865:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036442.239872:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036442.239878:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036442.239879:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036442.244599:0:252249:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036442.244606:0:252249:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036442.244611:0:252249:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1789036442.303067:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d62180] -> 12345-192.168.204.21@tcp (6) 00000800:00000200:0.0:1789036442.303075:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.21@tcp ip 192.168.204.21:988 00000800:00000200:0.0:1789036442.303097:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d892543c00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:3.0:1789036442.303315:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1789036442.303321:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.303333:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036442.303337:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.303339:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036442.945583:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036442.945598:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789036442.945610:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f7a300. 00000020:00000040:1.0:1789036442.945617:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d8861f3800 refcount=4 obd=MGC192.168.204.121@tcp 00000100:00000001:1.0:1789036442.945623:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036442.945626:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036442.945634:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036442.945644:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2ab00. 00000100:00000001:1.0:1789036442.945653:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.945670:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036442.945685:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036442.945690:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.945698:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041a5e0ff] to pc [ptlrpcd_00_00+0] req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.946185:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.946200:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036442.946203:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1789036442.946213:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1789036442.946217:0:2394:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1789036442.946220:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.946230:0:2394:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036442.946233:0:2394:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.946235:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.946236:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.946239:0:2394:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036442.946247:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.946267:0:2394:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036442.946270:0:2394:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.946275:0:2394:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036442.946297:0:2394:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036442.946300:0:2394:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036442.946898:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f7a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2394:1875927346552960:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036442.946905:0:2394:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036442.946913:0:2394:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036442.946916:0:2394:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.946919:0:2394:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036442.946926:0:2394:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26c400. 02000000:00000001:1.0:1789036442.946928:0:2394:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036442.946937:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036442.946942:0:2394:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903e580. 00000400:00000010:1.0:1789036442.946951:0:2394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d4c8. 00000100:00000200:1.0:1789036442.946961:0:2394:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346552960, portal 25 00000100:00000001:1.0:1789036442.946963:0:2394:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036442.946965:0:2394:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267937024 : -114518441614592 : ffff97d898f7a300) 00000100:00000040:1.0:1789036442.946973:0:2394:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.946983:0:2394:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036442.946986:0:2394:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036442.946989:0:2394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d000. 00000100:00000200:1.0:1789036442.946993:0:2394:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875927346552960, offset 0 00000400:00000200:1.0:1789036442.947001:0:2394:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036442.947012:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036442.947021:0:2394:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036442.947028:0:2394:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036442.947032:0:2394:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036442.947036:0:2394:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036442.947039:0:2394:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036442.947042:0:2394:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556990:556990:256:4294967295] 192.168.204.121@tcp LPNI seq info [556944:556944:8:4294967295] 00000400:00000200:1.0:1789036442.947046:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036442.947072:0:2394:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036442.947078:0:2394:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036442.947093:0:2394:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2a800. 00000800:00000200:1.0:1789036442.947102:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036442.947110:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036442.947115:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2a800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036442.947132:0:2394:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036442.947136:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036442.947140:0:2394:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 2 is 4307527897 00000100:00000010:1.0:1789036442.947148:0:2394:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8b183c4e0. 00000100:00000001:1.0:1789036442.947157:0:2394:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1789036442.947166:0:2394:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 3 is 4307527897 00000100:00000010:1.0:1789036442.947170:0:2394:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8856b2480. 00000100:00000001:1.0:1789036442.947174:0:2394:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1789036442.947177:0:2394:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036442.947181:0:2394:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.947185:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.947189:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1789036442.947209:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f7bb80. 00000020:00000040:1.0:1789036442.947215:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d887edf000 refcount=6 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000100:00000001:1.0:1789036442.947219:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036442.947221:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036442.947224:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036442.947230:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2a400. 00000100:00000001:1.0:1789036442.947236:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.947241:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036442.947263:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036442.947267:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.947271:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000858a87ca] to pc [ptlrpcd_00_01+1] req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.947289:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.947292:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036442.947302:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1789036442.947613:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1789036442.947617:0:2393:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1789036442.947619:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789036442.947625:0:2393:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1789036442.947629:0:2393:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789036442.947631:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789036442.947632:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789036442.947634:0:2393:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1789036442.947642:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789036442.947650:0:2393:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1789036442.947652:0:2393:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789036442.947655:0:2393:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1789036442.947658:0:2393:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789036442.947660:0:2393:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789036442.947664:0:2393:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f7bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346553088:192.168.204.121@tcp:400:kworker.0 00000100:00000001:0.0:1789036442.947668:0:2393:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1789036442.947672:0:2393:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1789036442.947673:0:2393:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789036442.947675:0:2393:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1789036442.947681:0:2393:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d886d8cc00. 02000000:00000001:0.0:1789036442.947683:0:2393:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1789036442.947689:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:0.0:1789036442.947692:0:2393:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b9ee8580. 00000400:00000010:0.0:1789036442.947697:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ba283a18. 00000100:00000200:0.0:1789036442.947701:0:2393:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553088, portal 10 00000100:00000001:0.0:1789036442.947703:0:2393:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1789036442.947705:0:2393:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267943296 : -114518441608320 : ffff97d898f7bb80) 00000100:00000040:0.0:1789036442.947713:0:2393:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789036442.947721:0:2393:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789036442.947723:0:2393:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:0.0:1789036442.947734:0:2393:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ba283b28. 00000100:00000200:0.0:1789036442.947738:0:2393:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346553088, offset 0 00000400:00000200:0.0:1789036442.947743:0:2393:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:0.0:1789036442.947748:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:0.0:1789036442.947752:0:2393:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:0.0:1789036442.947756:0:2393:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1789036442.947759:0:2393:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:0.0:1789036442.947761:0:2393:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1789036442.947764:0:2393:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:0.0:1789036442.947766:0:2393:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556991:556991:255:4294967295] 192.168.204.121@tcp LPNI seq info [556945:556945:7:4294967295] 00000400:00000200:0.0:1789036442.947770:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:0.0:1789036442.947777:0:2393:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:0.0:1789036442.947781:0:2393:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:0.0:1789036442.947786:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d890dd5000. 00000800:00000200:0.0:1789036442.947790:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:0.0:1789036442.947795:0:2393:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:0.0:1789036442.947799:0:2393:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d890dd5000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789036442.947802:0:2393:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789036442.947807:0:2393:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: using a timeout of 25705 usecs (26 jiffies) 00000400:00000200:0.0:1789036442.947810:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:0.0:1789036442.947813:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307524126 00000100:00000001:0.0:1789036442.947838:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307527897 00000100:00000001:0.0:1789036442.947843:0:2393:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307524126 00000100:00000001:0.0:1789036442.947845:0:2393:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307527897 00000100:00000001:0.0:1789036442.947847:0:2393:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789036442.947849:0:2393:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789036442.947851:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789036442.947853:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.948376:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.949589:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.949596:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.949603:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1789036442.949629:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f78700. 00000020:00000040:1.0:1789036442.949637:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d887edb000 refcount=4 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000100:00000001:1.0:1789036442.949643:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036442.949647:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036442.949653:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036442.949663:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2a200. 00000100:00000001:1.0:1789036442.949670:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.949685:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036442.949698:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036442.949703:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.949709:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a80e34e] to pc [ptlrpcd_00_00+0] req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.949737:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.949741:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789036442.949745:0:234282:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1789036442.949752:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.949754:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.949756:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036442.949758:0:2394:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1789036442.949760:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.949768:0:2394:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036442.949771:0:2394:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.949773:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.949774:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.949775:0:2394:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036442.949781:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.949812:0:2394:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036442.949829:0:2394:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.949834:0:2394:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036442.949837:0:2394:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036442.949840:0:2394:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036442.949845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f78700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2394:1875927346553216:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036442.949849:0:2394:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036442.949856:0:2394:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036442.949858:0:2394:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.949860:0:2394:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036442.949865:0:2394:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26c200. 02000000:00000001:1.0:1789036442.949867:0:2394:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036442.949877:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036442.949882:0:2394:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903e210. 00000400:00000010:1.0:1789036442.949890:0:2394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d5d8. 00000100:00000200:1.0:1789036442.949898:0:2394:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553216, portal 10 00000100:00000001:1.0:1789036442.949900:0:2394:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036442.949902:0:2394:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267929856 : -114518441621760 : ffff97d898f78700) 00000100:00000040:1.0:1789036442.949910:0:2394:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.949918:0:2394:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036442.949921:0:2394:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036442.949924:0:2394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62d880. 00000100:00000200:1.0:1789036442.949928:0:2394:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346553216, offset 0 00000400:00000200:1.0:1789036442.949935:0:2394:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036442.949943:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036442.949950:0:2394:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036442.949957:0:2394:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036442.949960:0:2394:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036442.949964:0:2394:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036442.949966:0:2394:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036442.949978:0:2394:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556992:556992:254:4294967295] 192.168.204.121@tcp LPNI seq info [556946:556946:6:4294967295] 00000400:00000200:1.0:1789036442.949983:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036442.949993:0:2394:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036442.949999:0:2394:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036442.950004:0:2394:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2af00. 00000800:00000200:1.0:1789036442.950011:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036442.950018:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036442.950023:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2af00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036442.950028:0:2394:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.950033:0:2394:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: using a timeout of 19616 usecs (20 jiffies) 00000400:00000200:1.0:1789036442.950036:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036442.950038:0:2394:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307524121 00000100:00000001:1.0:1789036442.950042:0:2394:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307527897 00000100:00000001:1.0:1789036442.950044:0:2394:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307524121 00000100:00000001:1.0:1789036442.950046:0:2394:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307527897 00000100:00000001:1.0:1789036442.950062:0:2394:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036442.950065:0:2394:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.950068:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.950070:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1789036442.950081:0:234282:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d898f7b100. 00000020:00000040:1.0:1789036442.950085:0:234282:0:(genops.c:1209:class_import_get()) import ffff97d887d65000 refcount=4 obd=lustre-OST0000-osc-ffff97d8b9387800 00000100:00000001:1.0:1789036442.950088:0:234282:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789036442.950090:0:234282:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789036442.950092:0:234282:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789036442.950096:0:234282:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d889b2ae00. 00000100:00000001:1.0:1789036442.950099:0:234282:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.950104:0:234282:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789036442.950111:0:234282:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789036442.950119:0:234282:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.950124:0:234282:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0348126] to pc [ptlrpcd_00_01+1] req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.950137:0:234282:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.950140:0:234282:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1789036442.950144:0:234282:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1789036442.950152:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.950154:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.950155:0:2394:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1789036442.950159:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.950165:0:2394:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1789036442.950167:0:2394:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.950169:0:2394:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1789036442.950170:0:2394:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1789036442.950172:0:2394:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1789036442.950175:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d898f7b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2394:1875927346553344:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036442.950179:0:2394:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1789036442.950181:0:2394:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1789036442.950199:0:2394:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.950200:0:2394:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1789036442.950204:0:2394:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d8ab26d400. 02000000:00000001:1.0:1789036442.950205:0:2394:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1789036442.950209:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:1.0:1789036442.950212:0:2394:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d8b903ed10. 00000400:00000010:1.0:1789036442.950216:0:2394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62dd48. 00000100:00000200:1.0:1789036442.950219:0:2394:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553344, portal 4 00000100:00000001:1.0:1789036442.950221:0:2394:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1789036442.950222:0:2394:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555267940608 : -114518441611008 : ffff97d898f7b100) 00000100:00000040:1.0:1789036442.950228:0:2394:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.950247:0:2394:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1789036442.950249:0:2394:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:1.0:1789036442.950262:0:2394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d8ac62dc38. 00000100:00000200:1.0:1789036442.950265:0:2394:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875927346553344, offset 0 00000400:00000200:1.0:1789036442.950270:0:2394:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:1.0:1789036442.950274:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036442.950278:0:2394:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:1.0:1789036442.950281:0:2394:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1789036442.950283:0:2394:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:1.0:1789036442.950285:0:2394:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1789036442.950288:0:2394:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:1.0:1789036442.950291:0:2394:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556993:556993:253:4294967295] 192.168.204.121@tcp LPNI seq info [556947:556947:5:4294967295] 00000400:00000200:1.0:1789036442.950295:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:1.0:1789036442.950301:0:2394:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:1.0:1789036442.950305:0:2394:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:1.0:1789036442.950309:0:2394:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d889b2ac00. 00000800:00000200:1.0:1789036442.950314:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:1.0:1789036442.950318:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:1.0:1789036442.950321:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d889b2ac00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1789036442.950324:0:2394:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.950328:0:2394:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: using a timeout of 102667 usecs (103 jiffies) 00000400:00000200:1.0:1789036442.950331:0:2394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:1.0:1789036442.950333:0:2394:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307524204 00000100:00000001:1.0:1789036442.950335:0:2394:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307527897 00000100:00000001:1.0:1789036442.950337:0:2394:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307524204 00000100:00000001:1.0:1789036442.950340:0:2394:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307527897 00000100:00000001:1.0:1789036442.950342:0:2394:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036442.950344:0:2394:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.950362:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.950365:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.950369:0:2394:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036442.950370:0:2394:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1789036442.950372:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.950374:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.950375:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.950377:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.950387:0:234282:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12861) 00000800:00000200:3.0:1789036442.951563:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036442.951572:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2a800. 00000400:00000200:3.0:1789036442.951579:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.951596:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036442.951606:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d000 00000400:00000010:3.0:1789036442.951609:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d000. 00000100:00000001:3.0:1789036442.951615:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036442.951623:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951636:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036442.951640:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951647:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036442.951648:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036442.951665:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036442.951667:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d890dd5000. 00000400:00000200:3.0:1789036442.951670:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.951674:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036442.951676:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ba283b28 00000400:00000010:3.0:1789036442.951678:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ba283b28. 00000100:00000001:3.0:1789036442.951681:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036442.951685:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951692:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036442.951696:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951702:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036442.951703:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036442.951712:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036442.951713:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2af00. 00000400:00000200:3.0:1789036442.951716:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.951719:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036442.951721:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d880 00000400:00000010:3.0:1789036442.951723:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d880. 00000100:00000001:3.0:1789036442.951726:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036442.951729:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951745:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036442.951747:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951753:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036442.951755:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789036442.951778:0:2387:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789036442.951781:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d889b2ac00. 00000400:00000200:3.0:1789036442.951784:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.951788:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:3.0:1789036442.951791:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62dc38 00000400:00000010:3.0:1789036442.951792:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62dc38. 00000100:00000001:3.0:1789036442.951794:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789036442.951798:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951805:0:2387:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036442.951808:0:2387:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.951813:0:2387:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036442.951847:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1789036442.955855:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789036442.955867:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789036442.955895:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789036442.955911:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1789036442.967355:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.967510:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036442.967562:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.967568:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.967580:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.967591:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036442.967603:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.967609:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.967615:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.967620:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1da100 00000400:00000200:3.0:1789036442.967627:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.967635:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392ed [1] + 224 00000800:00000001:3.0:1789036442.967653:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.967658:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.967662:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.967668:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036442.967678:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036442.967690:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036442.967703:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789036442.967781:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789036442.967787:0:2393:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036442.968684:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036442.968706:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.968715:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036442.968720:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.968723:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.968732:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.969235:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036442.969248:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.969262:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.969268:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.969274:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1da180 00000400:00000200:3.0:1789036442.969280:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.969288:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392f5 [1] + 224 00000800:00000001:3.0:1789036442.969297:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.969311:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.969315:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.969320:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036442.969326:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036442.969337:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036442.969349:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.970329:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.970336:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036442.970533:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036442.970549:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.970556:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036442.977588:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.977610:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036442.977616:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.977620:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.977632:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.977643:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036442.977654:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.977660:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000001:0.0:1789036442.978905:0:2393:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1789036442.978914:0:2393:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1789036442.978919:0:2393:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b9ee8580. 00000400:00000200:0.0:1789036442.978928:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ba283a18 00000400:00000010:0.0:1789036442.978931:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ba283a18. 00000100:00000001:0.0:1789036442.978937:0:2393:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1789036442.978952:0:2393:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1789036442.978966:0:2393:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789036442.978975:0:2393:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1789036442.978977:0:2393:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789036442.978980:0:2393:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1789036442.978985:0:2393:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1789036442.978987:0:2393:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789036442.978990:0:2393:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789036442.978993:0:2393:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789036442.979004:0:2393:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edf460 time=103 v=60 (60 61 61 62) 00000100:00001000:0.0:1789036442.979008:0:2393:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edf418 time=139 v=5 (1 1 0 0) 00000100:00000001:0.0:1789036442.979011:0:2393:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1789036442.979013:0:2393:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789036442.979016:0:2393:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1789036442.979020:0:2393:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789036442.979025:0:2393:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1789036442.979027:0:2393:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff97d8b9387800: skip recheck: last_committed 30064771460 00000100:00000001:0.0:1789036442.979029:0:2393:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1789036442.979031:0:2393:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789036442.979036:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789036442.979043:0:2393:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789036442.979047:0:2393:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1789036442.979067:0:2393:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1789036442.979073:0:2393:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789036442.979080:0:2393:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f7bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2393:1875927346553088:192.168.204.121@tcp:400:kworker.0 00000100:00000001:0.0:1789036442.979086:0:2393:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789036442.979090:0:2393:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1789036442.979093:0:2393:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f7bb80 x1875927346553088/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789036442.979101:0:2393:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1789036442.979104:0:2393:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1789036442.979109:0:2393:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d886d8cc00. 02000000:00000001:0.0:1789036442.979113:0:2393:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1789036442.979116:0:2393:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:0.0:1789036442.979118:0:2393:0:(genops.c:1222:class_import_put()) import ffff97d887edf000 refcount=5 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000020:00000001:0.0:1789036442.979120:0:2393:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:0.0:1789036442.979123:0:2393:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2a400. 02000000:00000001:0.0:1789036442.979127:0:2393:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1789036442.979129:0:2393:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1789036442.979131:0:2393:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f7bb80. 00000100:00000001:0.0:1789036442.979135:0:2393:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1789036442.979137:0:2393:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789036442.979140:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789036442.979150:0:2393:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1789036442.979152:0:2393:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789036442.979155:0:2393:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789036442.979159:0:2393:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1789036442.979517:0:2394:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036442.979521:0:2394:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036442.979524:0:2394:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903e210. 00000400:00000200:1.0:1789036442.979529:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d5d8 00000400:00000010:1.0:1789036442.979531:0:2394:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d5d8. 00000100:00000001:1.0:1789036442.979535:0:2394:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036442.979542:0:2394:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036442.979552:0:2394:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.979560:0:2394:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036442.979561:0:2394:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.979563:0:2394:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036442.979565:0:2394:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036442.979566:0:2394:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036442.979568:0:2394:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.979570:0:2394:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036442.979575:0:2394:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edb460 time=103 v=60 (60 61 61 62) 00000100:00001000:1.0:1789036442.979578:0:2394:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edb418 time=116 v=5 (1 1 0 0) 00000100:00000001:1.0:1789036442.979581:0:2394:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036442.979582:0:2394:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036442.979584:0:2394:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1789036442.979588:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1789036442.979594:0:2394:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.979604:0:2394:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1789036442.979606:0:2394:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff97d8b9387800: skip recheck: last_committed 21474836540 00000100:00000001:1.0:1789036442.979608:0:2394:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1789036442.979609:0:2394:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.979613:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.979620:0:2394:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036442.979623:0:2394:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036442.979624:0:2394:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036442.979627:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036442.979634:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f78700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2394:1875927346553216:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036442.979639:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.979641:0:2394:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036442.979644:0:2394:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f78700 x1875927346553216/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036527 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.979650:0:2394:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036442.979652:0:2394:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036442.979654:0:2394:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26c200. 02000000:00000001:1.0:1789036442.979656:0:2394:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036442.979658:0:2394:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036442.979659:0:2394:0:(genops.c:1222:class_import_put()) import ffff97d887edb000 refcount=3 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000020:00000001:1.0:1789036442.979661:0:2394:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036442.979662:0:2394:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2a200. 02000000:00000001:1.0:1789036442.979665:0:2394:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036442.979666:0:2394:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036442.979668:0:2394:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f78700. 00000100:00000001:1.0:1789036442.979670:0:2394:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036442.979671:0:2394:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.979674:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.979676:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.979678:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1789036442.979680:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.979685:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789036442.981023:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981035:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 4 MB=0x6aa255f1da200 00000400:00000200:3.0:1789036442.981042:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981049:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392fd [1] + 224 00000800:00000001:3.0:1789036442.981162:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.981249:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.981265:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.981271:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036442.981281:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036442.981291:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036442.981303:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.981720:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.981725:0:2394:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036442.981754:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036442.981769:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.981775:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036442.981779:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.981781:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.981788:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981794:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036442.981840:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981846:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981850:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981854:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 25 MB=0x6aa255f1da080 00000400:00000200:3.0:1789036442.981858:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036442.981863:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.121@tcp of length 224/224 into md 0x5392e5 [1] + 224 00000800:00000001:3.0:1789036442.981869:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.981876:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036442.981879:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036442.981883:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036442.981887:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036442.981894:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036442.981904:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036442.981912:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036442.981916:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036442.981920:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:1.0:1789036442.982098:0:2394:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036442.982101:0:2394:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036442.982105:0:2394:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903ed10. 00000400:00000200:1.0:1789036442.982110:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62dd48 00000400:00000010:1.0:1789036442.982113:0:2394:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62dd48. 00000100:00000001:1.0:1789036442.982117:0:2394:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036442.982123:0:2394:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036442.982132:0:2394:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.982139:0:2394:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036442.982141:0:2394:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.982144:0:2394:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036442.982148:0:2394:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036442.982151:0:2394:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036442.982153:0:2394:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.982156:0:2394:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036442.982166:0:2394:0:(import.c:2196:obd_at_measure()) add 36 to ffff97d887d65460 time=103 v=36 (36 5 5 5) 00000100:00001000:1.0:1789036442.982170:0:2394:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887d65418 time=31 v=5 (1 1 1 1) 00000100:00000001:1.0:1789036442.982173:0:2394:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036442.982175:0:2394:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036442.982179:0:2394:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1789036442.983980:0:2394:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.983996:0:2394:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1789036442.983999:0:2394:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff97d8b9387800: skip recheck: last_committed 42949695659 00000100:00000001:1.0:1789036442.984002:0:2394:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1789036442.984005:0:2394:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.984020:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.984030:0:2394:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036442.984034:0:2394:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036442.984037:0:2394:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036442.984066:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036442.984075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f7b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2394:1875927346553344:192.168.204.121@tcp:400:kworker.0 00000400:00000001:1.0:1789036442.984091:0:2394:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789036442.984093:0:2394:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789036442.984097:0:2394:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d8b903e580. 00000400:00000200:1.0:1789036442.984104:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d8ac62d4c8 00000400:00000010:1.0:1789036442.984107:0:2394:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d8ac62d4c8. 00000100:00000001:1.0:1789036442.984112:0:2394:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789036442.984117:0:2394:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789036442.984126:0:2394:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.984134:0:2394:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789036442.984136:0:2394:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.984139:0:2394:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789036442.984143:0:2394:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789036442.984146:0:2394:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789036442.984149:0:2394:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789036442.984152:0:2394:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789036442.984160:0:2394:0:(import.c:2196:obd_at_measure()) add 5 to ffff97d8861f3c60 time=103 v=5 (5 5 5 5) 00000100:00001000:1.0:1789036442.984164:0:2394:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d8861f3c18 time=139 v=5 (1 1 0 0) 00000100:00000001:1.0:1789036442.984166:0:2394:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789036442.984168:0:2394:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789036442.984171:0:2394:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1789036442.984173:0:2394:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.984175:0:2394:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789036442.984178:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.984264:0:2394:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789036442.984267:0:2394:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789036442.984268:0:2394:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789036442.984279:0:2394:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789036442.984287:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d898f7a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2394:1875927346552960:192.168.204.121@tcp:400:kworker.0 00000100:00000001:1.0:1789036442.984292:0:2394:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.984297:0:2394:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036442.984301:0:2394:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f7b100 x1875927346553344/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036497 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.985038:0:2394:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036442.985043:0:2394:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036442.985047:0:2394:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26d400. 02000000:00000001:1.0:1789036442.985054:0:2394:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036442.985057:0:2394:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036442.985059:0:2394:0:(genops.c:1222:class_import_put()) import ffff97d887d65000 refcount=3 obd=lustre-OST0000-osc-ffff97d8b9387800 00000020:00000001:1.0:1789036442.985063:0:2394:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036442.985066:0:2394:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2ae00. 02000000:00000001:1.0:1789036442.985070:0:2394:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036442.985073:0:2394:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036442.985075:0:2394:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f7b100. 00000100:00000001:1.0:1789036442.985079:0:2394:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036442.985081:0:2394:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.985083:0:2394:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789036442.985089:0:2394:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d898f7a300 x1875927346552960/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036458 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789036442.985096:0:2394:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789036442.985098:0:2394:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789036442.985099:0:2394:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d8ab26c400. 02000000:00000001:1.0:1789036442.985101:0:2394:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789036442.985102:0:2394:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789036442.985103:0:2394:0:(genops.c:1222:class_import_put()) import ffff97d8861f3800 refcount=3 obd=MGC192.168.204.121@tcp 00000020:00000001:1.0:1789036442.985105:0:2394:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789036442.985107:0:2394:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d889b2ab00. 02000000:00000001:1.0:1789036442.985109:0:2394:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789036442.985110:0:2394:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789036442.985111:0:2394:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d898f7a300. 00000100:00000001:1.0:1789036442.985123:0:2394:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789036442.985124:0:2394:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.985127:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789036442.985137:0:2394:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789036442.985139:0:2394:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789036442.985143:0:2394:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789036442.985146:0:2394:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036443.267199:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036443.267211:0:274228:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1789036443.267213:0:274228:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1789036443.267217:0:274228:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036443.267222:0:274228:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1789036443.267225:0:274228:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036443.267227:0:274228:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036443.267230:0:274228:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036443.267235:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036443.267236:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789036443.267272:0:236343:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789036443.267275:0:236343:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789036443.267279:0:236343:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036444.288264:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036444.288271:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036444.288277:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036444.288285:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036444.289334:0:248124:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036444.289339:0:248124:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036444.289343:0:248124:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1789036445.311018:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036445.311029:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036445.311036:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036445.311037:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036445.311098:0:242419:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036445.311102:0:242419:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036445.311105:0:242419:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:2.0:1789036445.838043:0:278929:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036446.338367:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036446.338379:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036446.338619:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036446.338621:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036446.338785:0:236342:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036446.338790:0:236342:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036446.338810:0:236342:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:00000001:2.0:1789036446.515204:0:278965:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:2.0:1789036446.515213:0:278965:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity test_812a: @@@@@@ FAIL: client: import is not in CONNECTING state after 50 00000001:00000001:2.0:1789036446.528553:0:278965:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00000100:00000040:3.0:1789036446.784538:0:252176:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712cb30 (cpu 3) has reached its deadline 4307527897, at 4307527937 00000100:00000010:3.0:1789036446.784560:0:252176:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8856b2480. 00000100:00000040:2.0:1789036446.785780:0:274228:0:(connection.c:75:cpu_latency_work()) work item of ffff97d88712ca20 (cpu 2) has reached its deadline 4307527897, at 4307527938 00000100:00000010:2.0:1789036446.785793:0:274228:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d8b183c4e0. 00010000:00000001:2.0:1789036447.362204:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036447.362213:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036447.362220:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036447.362221:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789036447.365914:0:252249:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789036447.365923:0:252249:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789036447.365929:0:252249:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1789036448.068343:0:261542:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036448.068353:0:261542:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036448.068364:0:261542:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac507800. 00000020:00000040:2.0:1789036448.068370:0:261542:0:(genops.c:1209:class_import_get()) import ffff97d8861f3800 refcount=4 obd=MGC192.168.204.121@tcp 00000100:00000001:2.0:1789036448.068375:0:261542:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036448.068378:0:261542:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036448.068382:0:261542:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036448.068390:0:261542:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d000. 00000100:00000001:2.0:1789036448.068398:0:261542:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068409:0:261542:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406->MGS req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036448.068420:0:261542:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036448.068425:0:261542:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068430:0:261542:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d61ba5d] to pc [ptlrpcd_01_01+1] req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068477:0:261542:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068480:0:261542:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036448.068483:0:261542:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036448.068486:0:261542:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac506680. 00000020:00000040:2.0:1789036448.068488:0:261542:0:(genops.c:1209:class_import_get()) import ffff97d887edf000 refcount=6 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000100:00000001:2.0:1789036448.068490:0:261542:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036448.068491:0:261542:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036448.068493:0:261542:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036448.068497:0:261542:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d800. 00000100:00000001:2.0:1789036448.068500:0:261542:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068505:0:261542:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0000_UUID req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036448.068512:0:261542:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036448.068514:0:261542:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068518:0:261542:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145566a4] to pc [ptlrpcd_01_00+0] req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068525:0:261542:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068527:0:261542:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036448.068531:0:261542:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036448.068533:0:261542:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac506a00. 00000020:00000040:2.0:1789036448.068535:0:261542:0:(genops.c:1209:class_import_get()) import ffff97d887edb000 refcount=4 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000100:00000001:2.0:1789036448.068537:0:261542:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036448.068538:0:261542:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036448.068540:0:261542:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036448.068544:0:261542:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d600. 00000100:00000001:2.0:1789036448.068547:0:261542:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068550:0:261542:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-MDT0001_UUID req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036448.068559:0:261542:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036448.068567:0:261542:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068572:0:261542:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009160264f] to pc [ptlrpcd_01_01+1] req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068578:0:261542:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068580:0:261542:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789036448.068583:0:261542:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789036448.068586:0:261542:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d8ac504a80. 00000020:00000040:2.0:1789036448.068588:0:261542:0:(genops.c:1209:class_import_get()) import ffff97d887d65000 refcount=4 obd=lustre-OST0000-osc-ffff97d8b9387800 00000100:00000001:2.0:1789036448.068590:0:261542:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789036448.068592:0:261542:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789036448.068593:0:261542:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789036448.068597:0:261542:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff97d886c6d300. 00000100:00000001:2.0:1789036448.068600:0:261542:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068603:0:261542:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0000_UUID req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789036448.068609:0:261542:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789036448.068611:0:261542:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068614:0:261542:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d50981d9] to pc [ptlrpcd_01_00+0] req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068620:0:261542:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.068622:0:261542:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1789036448.068626:0:261542:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7d763aff-8b7a-44d5-b6db-92551ac67e1a->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1789036448.068630:0:261542:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12866) 00000100:00000001:2.0:1789036448.068645:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036448.068649:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036448.068652:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036448.068657:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068663:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036448.068665:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.068675:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036448.068677:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036448.068680:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036448.068684:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac507800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2396:1875927346553472:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036448.068688:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036448.068693:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036448.068695:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.068698:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036448.068701:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d88b1a1400. 02000000:00000001:2.0:1789036448.068704:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036448.068711:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036448.068715:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb1160. 00000400:00000010:2.0:1789036448.068722:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b4c8. 00000100:00000200:2.0:1789036448.068730:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553472, portal 25 00000100:00000001:2.0:1789036448.068732:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036448.068734:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592525824 : -114518117025792 : ffff97d8ac507800) 00000100:00000040:2.0:1789036448.068739:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068745:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036448.068747:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036448.068749:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739bcc0. 00000100:00000200:2.0:1789036448.068752:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875927346553472, offset 0 00000400:00000200:2.0:1789036448.068758:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036448.068765:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.068773:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036448.068779:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036448.068783:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036448.068786:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036448.068789:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036448.068792:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556994:556994:256:4294967295] 192.168.204.121@tcp LPNI seq info [556948:556948:8:4294967295] 00000400:00000200:2.0:1789036448.068797:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.068806:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036448.068838:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036448.068843:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6dc00. 00000800:00000200:2.0:1789036448.068850:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036448.068857:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036448.068861:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6dc00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036448.068873:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036448.068878:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036448.068880:0:2396:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 2 is 4307533017 00000100:00000010:2.0:1789036448.068884:0:2396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8ac080f60. 00000100:00000001:2.0:1789036448.068891:0:2396:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1789036448.068896:0:2396:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS new deadline estimation for cpu 3 is 4307533017 00000100:00000010:2.0:1789036448.068899:0:2396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d8ac080360. 00000100:00000001:2.0:1789036448.068902:0:2396:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.121@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1789036448.068904:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036448.068907:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.068910:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036448.068913:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068919:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036448.068920:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.068923:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036448.068924:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036448.068926:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036448.068928:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346553728:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036448.068931:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036448.068933:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036448.068935:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.068936:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036448.068939:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d88b1a1800. 02000000:00000001:2.0:1789036448.068941:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036448.068945:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036448.068953:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb1bb0. 00000400:00000010:2.0:1789036448.068956:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739be58. 00000100:00000200:2.0:1789036448.068958:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553728, portal 10 00000100:00000001:2.0:1789036448.068960:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036448.068961:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592522240 : -114518117029376 : ffff97d8ac506a00) 00000100:00000040:2.0:1789036448.068968:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.068974:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036448.068977:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036448.068978:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b770. 00000100:00000200:2.0:1789036448.068980:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346553728, offset 0 00000400:00000200:2.0:1789036448.068984:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036448.068988:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.068991:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036448.068994:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036448.068996:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036448.068998:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036448.068999:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036448.069002:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556995:556995:255:4294967295] 192.168.204.121@tcp LPNI seq info [556949:556949:7:4294967295] 00000400:00000200:2.0:1789036448.069005:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.069011:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036448.069014:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036448.069017:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6d100. 00000800:00000200:2.0:1789036448.069021:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036448.069025:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036448.069027:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6d100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036448.069029:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.069032:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: using a timeout of 19616 usecs (20 jiffies) 00000400:00000200:2.0:1789036448.069035:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036448.069037:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307529239 00000100:00000001:2.0:1789036448.069040:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307533017 00000100:00000001:2.0:1789036448.069047:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307529239 00000100:00000001:2.0:1789036448.069049:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307533017 00000100:00000001:2.0:1789036448.069051:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036448.069053:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069055:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069058:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036448.069065:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789036448.069067:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789036448.069069:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036448.069070:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036448.069072:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036448.069075:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:2.0:1789036448.069076:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036448.069078:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789036448.069079:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036448.069081:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036448.069082:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036448.069083:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036448.069087:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069094:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036448.069095:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.069097:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036448.069099:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036448.069101:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036448.069104:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346553600:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036448.069107:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036448.069109:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036448.069110:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.069112:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036448.069115:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d88b1a0c00. 02000000:00000001:2.0:1789036448.069117:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036448.069119:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036448.069122:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb10b0. 00000400:00000010:2.0:1789036448.069130:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739bf68. 00000100:00000200:2.0:1789036448.069132:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553600, portal 10 00000100:00000001:2.0:1789036448.069134:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036448.069136:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592521344 : -114518117030272 : ffff97d8ac506680) 00000100:00000040:2.0:1789036448.069142:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069148:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036448.069150:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036448.069153:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b880. 00000100:00000200:2.0:1789036448.069166:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875927346553600, offset 0 00000400:00000200:2.0:1789036448.069170:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036448.069174:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.069178:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036448.069180:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036448.069183:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036448.069185:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036448.069187:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036448.069189:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556996:556996:254:4294967295] 192.168.204.121@tcp LPNI seq info [556950:556950:6:4294967295] 00000400:00000200:2.0:1789036448.069193:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.069198:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036448.069202:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036448.069205:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6df00. 00000800:00000200:2.0:1789036448.069208:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036448.069212:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036448.069215:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6df00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036448.069217:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.069220:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: using a timeout of 25710 usecs (26 jiffies) 00000400:00000200:2.0:1789036448.069223:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036448.069225:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307529245 00000100:00000001:2.0:1789036448.069227:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307533017 00000100:00000001:2.0:1789036448.069235:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307529245 00000100:00000001:2.0:1789036448.069237:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307533017 00000100:00000001:2.0:1789036448.069238:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036448.069240:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069242:0:2396:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789036448.069247:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069253:0:2396:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789036448.069254:0:2396:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.069256:0:2396:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789036448.069258:0:2396:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789036448.069260:0:2396:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789036448.069262:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff97d8ac504a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346553856:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036448.069265:0:2396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789036448.069267:0:2396:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789036448.069269:0:2396:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.069270:0:2396:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789036448.069274:0:2396:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff97d89244c000. 02000000:00000001:2.0:1789036448.069276:0:2396:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789036448.069279:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000010:2.0:1789036448.069282:0:2396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d882fb1528. 00000400:00000010:2.0:1789036448.069286:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739b5d8. 00000100:00000200:2.0:1789036448.069289:0:2396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875927346553856, portal 4 00000100:00000001:2.0:1789036448.069290:0:2396:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789036448.069291:0:2396:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446629555592514176 : -114518117037440 : ffff97d8ac504a80) 00000100:00000040:2.0:1789036448.069297:0:2396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069303:0:2396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789036448.069305:0:2396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.121@tcp 00000400:00000010:2.0:1789036448.069307:0:2396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d88739bdd0. 00000100:00000200:2.0:1789036448.069309:0:2396:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875927346553856, offset 0 00000400:00000200:2.0:1789036448.069318:0:2396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.121@tcp 00000400:00000200:2.0:1789036448.069321:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.069325:0:2396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.121@tcp local destination 00000400:00000200:2.0:1789036448.069327:0:2396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789036448.069330:0:2396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.21@tcp 00000400:00000200:2.0:1789036448.069332:0:2396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.21@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789036448.069334:0:2396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.121@tcp 00000400:00000200:2.0:1789036448.069337:0:2396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.21@tcp NI seq info: [556997:556997:253:4294967295] 192.168.204.121@tcp LPNI seq info [556951:556951:5:4294967295] 00000400:00000200:2.0:1789036448.069341:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.069346:0:2396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp:) -> 192.168.204.121@tcp(192.168.204.121@tcp:192.168.204.121@tcp) : PUT try# 0 00000800:00000200:2.0:1789036448.069350:0:2396:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.121@tcp 00000800:00000010:2.0:1789036448.069353:0:2396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d886c6da00. 00000800:00000200:2.0:1789036448.069356:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d890d6fc00] -> 12345-192.168.204.121@tcp (5) 00000800:00000200:2.0:1789036448.069360:0:2396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:2.0:1789036448.069363:0:2396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d886c6da00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789036448.069365:0:2396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.069368:0:2396:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: using a timeout of 102617 usecs (103 jiffies) 00000400:00000200:2.0:1789036448.069371:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000100:00000001:2.0:1789036448.069372:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 2 is 4307529322 00000100:00000001:2.0:1789036448.069375:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 2, keep current deadline 4307533017 00000100:00000001:2.0:1789036448.069377:0:2396:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS new deadline estimation for cpu 3 is 4307529322 00000100:00000001:2.0:1789036448.069379:0:2396:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d8b9387800: PM QoS request active for cpu 3, keep current deadline 4307533017 00000100:00000001:2.0:1789036448.069380:0:2396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036448.069382:0:2396:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069384:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069386:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069399:0:2395:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036448.069401:0:2395:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789036448.069435:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036448.069439:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6dc00. 00000400:00000200:2.0:1789036448.069451:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036448.069456:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036448.069460:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739bcc0 00000400:00000010:2.0:1789036448.069463:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739bcc0. 00000100:00000001:2.0:1789036448.069467:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036448.069471:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069479:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036448.069482:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069488:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069490:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789036448.069504:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036448.069506:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6d100. 00000400:00000200:2.0:1789036448.069508:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036448.069511:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036448.069513:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b770 00000400:00000010:2.0:1789036448.069515:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b770. 00000100:00000001:2.0:1789036448.069517:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036448.069520:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069527:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036448.069529:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069535:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069537:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789036448.069544:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036448.069546:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6df00. 00000400:00000200:2.0:1789036448.069548:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036448.069551:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036448.069553:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b880 00000400:00000010:2.0:1789036448.069555:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b880. 00000100:00000001:2.0:1789036448.069557:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036448.069565:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069571:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036448.069574:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069579:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069580:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789036448.069643:0:2388:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789036448.069645:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d886c6da00. 00000400:00000200:2.0:1789036448.069648:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036448.069651:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000400:00000200:2.0:1789036448.069653:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739bdd0 00000400:00000010:2.0:1789036448.069655:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739bdd0. 00000100:00000001:2.0:1789036448.069657:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789036448.069660:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069666:0:2388:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036448.069669:0:2388:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.069675:0:2388:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.069676:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:3.0:1789036448.074260:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.074277:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036448.074281:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.074284:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036448.074293:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.074300:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036448.074309:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.074313:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.074319:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.074323:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 25 MB=0x6aa255f1da280 00000400:00000200:3.0:1789036448.074329:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.074349:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.121@tcp of length 224/224 into md 0x539305 [1] + 224 00000800:00000001:3.0:1789036448.074356:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.074362:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036448.074366:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036448.074370:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036448.074378:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036448.074388:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036448.074399:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.074424:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036448.074432:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.074436:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036448.074515:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036448.074519:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1789036448.074526:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789036448.074528:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789036448.074531:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb1160. 00000400:00000200:2.0:1789036448.074535:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b4c8 00000400:00000010:2.0:1789036448.074538:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b4c8. 00000100:00000001:2.0:1789036448.074541:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036448.074545:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789036448.074554:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.074571:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789036448.074573:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036448.074575:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789036448.074578:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789036448.074581:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789036448.074583:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789036448.074586:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789036448.074592:0:2396:0:(import.c:2196:obd_at_measure()) add 5 to ffff97d8861f3c60 time=109 v=5 (5 5 5 5) 00000100:00001000:2.0:1789036448.074596:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d8861f3c18 time=145 v=5 (1 1 0 0) 00000100:00000001:2.0:1789036448.074598:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789036448.074600:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036448.074602:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789036448.074604:0:2396:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.074607:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789036448.074612:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.074618:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789036448.074621:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789036448.074623:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789036448.074626:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789036448.074634:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac507800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:9bb1e375-e4f5-4920-9d1b-5b6c0fdb9406:2396:1875927346553472:192.168.204.121@tcp:400:kworker.0 00000100:00000001:2.0:1789036448.074639:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.074642:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789036448.074646:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac507800 x1875927346553472/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1789036464 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.074652:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789036448.074655:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789036448.074658:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d88b1a1400. 02000000:00000001:2.0:1789036448.074663:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789036448.074666:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789036448.074673:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d8861f3800 refcount=3 obd=MGC192.168.204.121@tcp 00000020:00000001:2.0:1789036448.074675:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789036448.074678:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d000. 02000000:00000001:2.0:1789036448.074682:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789036448.074684:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789036448.074686:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac507800. 00000100:00000001:2.0:1789036448.074690:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789036448.074691:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789036448.074694:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.074696:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789036448.074697:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789036448.074699:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789036448.074701:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789036448.078236:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078258:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036448.078262:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078265:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036448.078273:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078281:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036448.078293:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078298:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078302:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078307:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1da380 00000400:00000200:2.0:1789036448.078313:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078318:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x539315 [1] + 224 00000800:00000001:2.0:1789036448.078326:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078330:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036448.078333:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036448.078337:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036448.078343:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036448.078352:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036448.078363:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.078411:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036448.078424:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078428:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789036448.078430:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078431:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036448.078434:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078437:0:2388:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:2.0:1789036448.078441:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078443:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078446:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078448:0:2388:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 10 MB=0x6aa255f1da300 00000400:00000200:2.0:1789036448.078451:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:2.0:1789036448.078454:0:2388:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.121@tcp of length 224/224 into md 0x539325 [1] + 224 00000800:00000001:2.0:1789036448.078457:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078464:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789036448.078466:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789036448.078469:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:2.0:1789036448.078472:0:2388:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789036448.078476:0:2388:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789036448.078483:0:2388:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789036448.078489:0:2388:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789036448.078492:0:2388:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789036448.078495:0:2388:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.078519:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036448.078525:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789036448.078531:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036448.078533:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036448.078537:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb10b0. 00000400:00000200:3.0:1789036448.078542:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739bf68 00000400:00000010:3.0:1789036448.078545:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739bf68. 00000100:00000001:3.0:1789036448.078548:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036448.078554:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036448.078564:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.078571:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036448.078573:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.078575:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036448.078579:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036448.078581:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036448.078583:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036448.078586:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036448.078594:0:2396:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edf460 time=109 v=60 (60 61 61 62) 00000100:00001000:3.0:1789036448.078606:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edf418 time=145 v=5 (1 1 0 0) 00000100:00000001:3.0:1789036448.078609:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036448.078610:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036448.078613:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789036448.078617:0:2396:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.078620:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789036448.078622:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff97d8b9387800: skip recheck: last_committed 30064771460 00000100:00000001:3.0:1789036448.078624:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789036448.078626:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036448.078631:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.078639:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036448.078642:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036448.078643:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036448.078647:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036448.078655:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346553600:192.168.204.121@tcp:400:kworker.0 00000400:00000001:3.0:1789036448.078672:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036448.078674:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036448.078676:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb1bb0. 00000400:00000200:3.0:1789036448.078678:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739be58 00000400:00000010:3.0:1789036448.078680:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739be58. 00000100:00000001:3.0:1789036448.078682:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036448.078685:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036448.078693:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.078700:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036448.078701:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.078703:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036448.078704:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036448.078710:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036448.078711:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036448.078713:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036448.078718:0:2396:0:(import.c:2196:obd_at_measure()) add 60 to ffff97d887edb460 time=109 v=60 (60 61 61 62) 00000100:00001000:3.0:1789036448.078721:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887edb418 time=122 v=5 (1 1 0 0) 00000100:00000001:3.0:1789036448.078723:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036448.078724:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036448.078726:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1789036448.078730:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789036448.078737:0:2396:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.078739:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789036448.078740:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff97d8b9387800: skip recheck: last_committed 21474836540 00000100:00000001:3.0:1789036448.078741:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789036448.078742:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036448.078746:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.078752:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036448.078753:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036448.078755:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036448.078758:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036448.078765:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346553728:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036448.078769:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.078771:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036448.078775:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac506680 x1875927346553600/t0(0) o400->lustre-MDT0000-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.078781:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036448.078784:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036448.078787:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d88b1a0c00. 02000000:00000001:3.0:1789036448.078790:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036448.078796:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036448.078798:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887edf000 refcount=5 obd=lustre-MDT0000-mdc-ffff97d8b9387800 00000020:00000001:3.0:1789036448.078800:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036448.078803:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d800. 02000000:00000001:3.0:1789036448.078806:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036448.078808:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036448.078810:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac506680. 00000100:00000001:3.0:1789036448.078899:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036448.078902:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.078905:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036448.078909:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac506a00 x1875927346553728/t0(0) o400->lustre-MDT0001-mdc-ffff97d8b9387800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 0 dl 1789036533 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.078916:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036448.078918:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036448.078920:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d88b1a1800. 02000000:00000001:3.0:1789036448.078923:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036448.078925:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036448.078926:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887edb000 refcount=3 obd=lustre-MDT0001-mdc-ffff97d8b9387800 00000020:00000001:3.0:1789036448.078929:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036448.078930:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d600. 02000000:00000001:3.0:1789036448.078933:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036448.078934:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036448.078935:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac506a00. 00000100:00000001:3.0:1789036448.078937:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036448.078939:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.078942:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.078944:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036448.078945:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789036448.078946:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.078948:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1789036448.096038:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.096053:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789036448.096056:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.096059:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036448.096067:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.096075:0:2387:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.21@tcp(192.168.204.21@tcp) <- 192.168.204.121@tcp : PUT - for me 00000400:00000200:3.0:1789036448.096083:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.096094:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.096099:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.096103:0:2387:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.121@tcp of length 224 into portal 4 MB=0x6aa255f1da400 00000400:00000200:3.0:1789036448.096109:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.121@tcp to cpt 1 00000400:00000200:3.0:1789036448.096114:0:2387:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.121@tcp of length 224/224 into md 0x539335 [1] + 224 00000800:00000001:3.0:1789036448.096122:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.096129:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789036448.096132:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789036448.096136:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.21@tcp->192.168.204.121@tcp: PUT: OK 00000100:00000001:3.0:1789036448.096144:0:2387:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036448.096162:0:2387:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789036448.096173:0:2387:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.096188:0:2387:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789036448.096195:0:2387:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789036448.096199:0:2387:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.096213:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036448.096217:0:2396:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789036448.096224:0:2396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789036448.096226:0:2396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789036448.096228:0:2396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d882fb1528. 00000400:00000200:3.0:1789036448.096233:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d88739b5d8 00000400:00000010:3.0:1789036448.096235:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff97d88739b5d8. 00000100:00000001:3.0:1789036448.096238:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789036448.096242:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789036448.096251:0:2396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.096257:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789036448.096259:0:2396:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.096261:0:2396:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789036448.096269:0:2396:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789036448.096271:0:2396:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789036448.096273:0:2396:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789036448.096277:0:2396:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789036448.096290:0:2396:0:(import.c:2196:obd_at_measure()) add 36 to ffff97d887d65460 time=109 v=36 (36 5 5 5) 00000100:00001000:3.0:1789036448.096295:0:2396:0:(import.c:2196:obd_at_measure()) add 1 to ffff97d887d65418 time=37 v=5 (1 1 1 1) 00000100:00000001:3.0:1789036448.096297:0:2396:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789036448.096299:0:2396:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789036448.096301:0:2396:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789036448.096304:0:2396:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.096308:0:2396:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789036448.096310:0:2396:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff97d8b9387800: skip recheck: last_committed 42949695659 00000100:00000001:3.0:1789036448.096312:0:2396:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789036448.096314:0:2396:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789036448.096320:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.096326:0:2396:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789036448.096329:0:2396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789036448.096331:0:2396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789036448.096335:0:2396:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789036448.096342:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff97d8ac504a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7d763aff-8b7a-44d5-b6db-92551ac67e1a:2396:1875927346553856:192.168.204.121@tcp:400:kworker.0 00000100:00000001:3.0:1789036448.096347:0:2396:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.096350:0:2396:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789036448.096353:0:2396:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff97d8ac504a80 x1875927346553856/t0(0) o400->lustre-OST0000-osc-ffff97d8b9387800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 0 dl 1789036503 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789036448.096360:0:2396:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789036448.096362:0:2396:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789036448.096366:0:2396:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff97d89244c000. 02000000:00000001:3.0:1789036448.096369:0:2396:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789036448.096372:0:2396:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789036448.096374:0:2396:0:(genops.c:1222:class_import_put()) import ffff97d887d65000 refcount=3 obd=lustre-OST0000-osc-ffff97d8b9387800 00000020:00000001:3.0:1789036448.096376:0:2396:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789036448.096383:0:2396:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff97d886c6d300. 02000000:00000001:3.0:1789036448.096387:0:2396:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789036448.096389:0:2396:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789036448.096391:0:2396:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d8ac504a80. 00000100:00000001:3.0:1789036448.096394:0:2396:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789036448.096396:0:2396:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.096398:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789036448.096405:0:2396:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789036448.096407:0:2396:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789036448.096409:0:2396:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789036448.096411:0:2396:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1789036448.222089:0:279065:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036448.384250:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036448.384257:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789036448.384263:0:274228:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1789036448.384264:0:274228:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789036448.384598:0:236343:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789036448.384602:0:236343:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789036448.384622:0:236343:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 3300 lines, 3300 kept, 0 dropped, 0 bad.