00000100:00000040:2.0F:1789061120.859634:0:245520:0:(pinger.c:229:ptlrpc_pinger_process_import()) f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789061120.859655:0:245520:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789061120.859661:0:245520:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf46c038000. 00000020:00000040:2.0:1789061120.859667:0:245520:0:(genops.c:1209:class_import_get()) import ffff9bf442d05800 refcount=4 obd=MGC192.168.204.129@tcp 00000100:00000001:2.0:1789061120.859672:0:245520:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789061120.859674:0:245520:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789061120.859679:0:245520:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789061120.859687:0:245520:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf34dd7f700. 00000100:00000001:2.0:1789061120.859693:0:245520:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859704:0:245520:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789061120.859715:0:245520:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789061120.859720:0:245520:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859725:0:245520:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000667e6536] to pc [ptlrpcd_01_01+1] req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.859760:0:245520:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859763:0:245520:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789061120.859775:0:245520:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789061120.859778:0:245520:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf46c03b480. 00000020:00000040:2.0:1789061120.859780:0:245520:0:(genops.c:1209:class_import_get()) import ffff9bf448dc5000 refcount=9 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000100:00000001:2.0:1789061120.859783:0:245520:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789061120.859784:0:245520:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789061120.859785:0:245520:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789061120.859789:0:245520:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf34dd7fc00. 00000100:00000001:2.0:1789061120.859791:0:245520:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859796:0:245520:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789061120.859803:0:245520:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789061120.859805:0:245520:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859809:0:245520:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000571621f4] to pc [ptlrpcd_01_00+0] req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.859818:0:245520:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859825:0:245520:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789061120.859828:0:245520:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789061120.859831:0:245520:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf46c038a80. 00000020:00000040:2.0:1789061120.859833:0:245520:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6800 refcount=4 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000100:00000001:2.0:1789061120.859835:0:245520:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789061120.859837:0:245520:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789061120.859838:0:245520:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789061120.859842:0:245520:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf34dd7f300. 00000100:00000001:2.0:1789061120.859845:0:245520:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859849:0:245520:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789061120.859856:0:245520:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789061120.859858:0:245520:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859862:0:245520:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072c0fb29] to pc [ptlrpcd_01_01+1] req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.859868:0:245520:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859870:0:245520:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789061120.859874:0:245520:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1789061120.859877:0:245520:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf46c038700. 00000020:00000040:2.0:1789061120.859879:0:245520:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6000 refcount=4 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000100:00000001:2.0:1789061120.859880:0:245520:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1789061120.859881:0:245520:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1789061120.859883:0:245520:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1789061120.859886:0:245520:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf34dd7fb00. 00000100:00000001:2.0:1789061120.859888:0:245520:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859892:0:245520:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789061120.859898:0:245520:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1789061120.859900:0:245520:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859903:0:245520:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000073d6dc86] to pc [ptlrpcd_01_00+0] req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.859922:0:245520:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.859924:0:245520:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1789061120.859926:0:245520:0:(pinger.c:247:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1789061120.859931:0:245520:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12838) 00000100:00000001:2.0:1789061120.860080:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061120.860083:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061120.860086:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061120.860092:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.860100:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061120.860102:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.860106:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061120.860108:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061120.860111:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061120.860114:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf46c038000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2406:1875953296540288:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061120.860118:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061120.860123:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061120.860125:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.860128:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061120.860132:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf4472d5800. 02000000:00000001:2.0:1789061120.860133:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061120.860140:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061120.860145:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895108. 00000400:00000010:2.0:1789061120.860151:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ec38. 00000100:00000200:2.0:1789061120.860158:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296540288, portal 25 00000100:00000001:2.0:1789061120.860160:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061120.860162:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072819335168 : -110000890216448 : ffff9bf46c038000) 00000100:00000040:2.0:1789061120.860167:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.860173:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061120.860182:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061120.860185:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e5d8. 00000100:00000200:2.0:1789061120.860188:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875953296540288, offset 0 00000400:00000200:2.0:1789061120.860195:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061120.860203:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.860211:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061120.860215:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061120.860219:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061120.860222:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061120.860225:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061120.860228:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554644:554644:256:4294967295] 192.168.204.129@tcp LPNI seq info [554595:554595:8:4294967295] 00000400:00000200:2.0:1789061120.860231:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.860240:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061120.860245:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061120.860249:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f800. 00000800:00000200:2.0:1789061120.860256:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061120.860263:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061120.860267:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061120.860278:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061120.860281:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061120.860284:0:2406:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 2 is 4307504860 00000100:00000010:2.0:1789061120.860289:0:2406:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf479a68c60. 00000100:00000001:2.0:1789061120.860294:0:2406:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1789061120.860298:0:2406:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 3 is 4307504860 00000100:00000010:2.0:1789061120.860300:0:2406:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf479a680c0. 00000100:00000001:2.0:1789061120.860303:0:2406:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1789061120.860306:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.860309:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.860311:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061120.860315:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.860331:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061120.860332:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.860334:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061120.860336:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061120.860338:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061120.860341:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf46c038a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540544:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061120.860344:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061120.860346:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061120.860347:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.860349:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061120.860353:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1ec00. 02000000:00000001:2.0:1789061120.860355:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061120.860358:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061120.860361:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895478. 00000400:00000010:2.0:1789061120.860365:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ebb0. 00000100:00000200:2.0:1789061120.860367:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296540544, portal 10 00000100:00000001:2.0:1789061120.860370:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061120.860371:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072819337856 : -110000890213760 : ffff9bf46c038a80) 00000100:00000040:2.0:1789061120.860379:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.860385:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061120.860387:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061120.860389:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e440. 00000100:00000200:2.0:1789061120.860392:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296540544, offset 0 00000400:00000200:2.0:1789061120.860396:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061120.860399:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.860403:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061120.860406:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061120.860408:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061120.860411:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061120.860412:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061120.860420:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554645:554645:255:4294967295] 192.168.204.129@tcp LPNI seq info [554596:554596:7:4294967295] 00000400:00000200:2.0:1789061120.860438:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.860444:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061120.860449:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061120.860452:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7fd00. 00000800:00000200:2.0:1789061120.860881:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061120.860887:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061120.860890:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7fd00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061120.860894:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.860898:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: using a timeout of 22407 usecs (23 jiffies) 00000400:00000200:2.0:1789061120.860902:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061120.860904:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307501081 00000100:00000001:2.0:1789061120.860907:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307504860 00000100:00000001:2.0:1789061120.860909:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307501081 00000100:00000001:2.0:1789061120.860911:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307504860 00000100:00000001:2.0:1789061120.860913:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.860915:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.860918:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.860921:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.860928:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061120.860930:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789061120.860933:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061120.860934:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061120.860936:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061120.860939:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:2.0:1789061120.860940:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.860942:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061120.860943:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.860945:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061120.860946:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061120.860967:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061120.860973:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.860988:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061120.860990:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.860992:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061120.860994:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061120.860996:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061120.860999:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf46c03b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540416:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061120.861002:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061120.861005:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061120.861006:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.861007:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061120.861010:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1fc00. 02000000:00000001:2.0:1789061120.861012:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061120.861016:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061120.861018:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895318. 00000400:00000010:2.0:1789061120.861021:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ed48. 00000100:00000200:2.0:1789061120.861024:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296540416, portal 10 00000100:00000001:2.0:1789061120.861026:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061120.861028:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072819348608 : -110000890203008 : ffff9bf46c03b480) 00000100:00000040:2.0:1789061120.861033:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.861039:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061120.861041:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061120.861043:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e990. 00000100:00000200:2.0:1789061120.861046:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296540416, offset 0 00000400:00000200:2.0:1789061120.861050:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061120.861054:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.861058:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061120.861061:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061120.861063:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061120.861066:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061120.861068:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061120.861077:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554646:554646:254:4294967295] 192.168.204.129@tcp LPNI seq info [554597:554597:6:4294967295] 00000400:00000200:2.0:1789061120.861081:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.861088:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061120.861092:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061120.861097:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f500. 00000800:00000200:2.0:1789061120.861102:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061120.861106:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061120.861108:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061120.861111:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.861114:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: using a timeout of 29480 usecs (30 jiffies) 00000400:00000200:2.0:1789061120.861117:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061120.861118:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307501088 00000100:00000001:2.0:1789061120.861121:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307504860 00000100:00000001:2.0:1789061120.861122:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307501088 00000100:00000001:2.0:1789061120.861124:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307504860 00000100:00000001:2.0:1789061120.861126:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.861128:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.861129:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061120.861133:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.861139:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061120.861140:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.861142:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061120.861144:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061120.861146:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061120.861149:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf46c038700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540672:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061120.861152:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061120.861154:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061120.861156:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.861163:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061120.861166:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1f600. 02000000:00000001:2.0:1789061120.861168:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061120.861171:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061120.861174:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895268. 00000400:00000010:2.0:1789061120.861177:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ee58. 00000100:00000200:2.0:1789061120.861180:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296540672, portal 4 00000100:00000001:2.0:1789061120.861181:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061120.861182:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072819336960 : -110000890214656 : ffff9bf46c038700) 00000100:00000040:2.0:1789061120.861188:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.861195:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061120.861197:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061120.861199:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e330. 00000100:00000200:2.0:1789061120.861202:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875953296540672, offset 0 00000400:00000200:2.0:1789061120.861206:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061120.861209:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.861213:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061120.861215:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061120.861217:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061120.861219:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061120.861221:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061120.861223:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554647:554647:253:4294967295] 192.168.204.129@tcp LPNI seq info [554598:554598:5:4294967295] 00000400:00000200:2.0:1789061120.861227:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.861232:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061120.861236:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061120.861239:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f000. 00000800:00000200:2.0:1789061120.861242:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061120.861246:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061120.861248:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061120.861250:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.861258:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: using a timeout of 111207 usecs (112 jiffies) 00000400:00000200:2.0:1789061120.861261:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061120.861263:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307501170 00000100:00000001:2.0:1789061120.861265:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307504860 00000100:00000001:2.0:1789061120.861267:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307501170 00000100:00000001:2.0:1789061120.861269:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307504860 00000100:00000001:2.0:1789061120.861271:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.861273:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.861276:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.861278:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.861293:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061120.861296:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789061120.861343:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061120.861347:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f800. 00000400:00000200:2.0:1789061120.861353:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.861358:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061120.861364:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e5d8 00000400:00000010:2.0:1789061120.861366:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e5d8. 00000100:00000001:2.0:1789061120.861370:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061120.861375:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.861384:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.861387:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.861394:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.861396:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789061120.861407:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061120.861409:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7fd00. 00000400:00000200:2.0:1789061120.861412:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.861415:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061120.861417:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e440 00000400:00000010:2.0:1789061120.861419:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e440. 00000100:00000001:2.0:1789061120.863596:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061120.863611:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.863624:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.863628:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.863635:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.863637:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789061120.863673:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061120.863677:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f500. 00000400:00000200:2.0:1789061120.863682:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.863687:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061120.863692:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e990 00000400:00000010:2.0:1789061120.863695:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e990. 00000100:00000001:2.0:1789061120.863697:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061120.863701:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.863708:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.863711:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.863719:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.863720:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789061120.864117:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061120.864120:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f000. 00000400:00000200:2.0:1789061120.864124:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.864128:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061120.864132:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e330 00000400:00000010:2.0:1789061120.864134:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e330. 00000100:00000001:2.0:1789061120.864137:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061120.864142:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.864150:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.864165:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.864171:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.864173:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1789061120.868420:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868453:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061120.868457:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868460:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.868469:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868478:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061120.868486:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868491:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868496:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868500:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0380 00000400:00000200:2.0:1789061120.868506:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868512:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x5347f5 [1] + 224 00000800:00000001:2.0:1789061120.868519:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868524:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.868527:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.868532:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061120.868539:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.868550:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061120.868562:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.868579:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061120.868586:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868590:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061120.868592:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868594:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.868596:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868600:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061120.868612:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868615:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868618:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868621:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0300 00000400:00000200:2.0:1789061120.868625:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868628:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x534805 [1] + 224 00000800:00000001:2.0:1789061120.868633:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868636:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.868639:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.868642:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061120.868645:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.868650:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061120.868659:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.868668:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061120.868671:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868674:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061120.868676:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868678:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.868681:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868684:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061120.868687:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868690:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868692:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868695:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 25 MB=0x6aa2b69db0280 00000400:00000200:2.0:1789061120.868698:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.868702:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.129@tcp of length 224/224 into md 0x5347e5 [1] + 224 00000800:00000001:2.0:1789061120.868705:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868713:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.868714:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.868724:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061120.868728:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.868733:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061120.868752:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.868759:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061120.868764:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.868776:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.868793:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061120.868798:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1789061120.868805:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789061120.868807:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789061120.868810:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895318. 00000400:00000200:2.0:1789061120.868815:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ed48 00000400:00000010:2.0:1789061120.868818:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ed48. 00000100:00000001:2.0:1789061120.868821:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.868825:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789061120.868834:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.868840:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789061120.868843:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.868845:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789061120.868849:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789061120.868852:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789061120.868854:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.868857:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789061120.868869:0:2406:0:(import.c:2196:obd_at_measure()) add 60 to ffff9bf448dc5460 time=124 v=60 (60 60 60 60) 00000100:00001000:2.0:1789061120.868874:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc5418 time=101 v=5 (1 1 1 1) 00000100:00000001:2.0:1789061120.868877:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789061120.868879:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789061120.868882:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789061120.868886:0:2406:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.868896:0:2406:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789061120.868899:0:2406:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9bf4804ce000: skip recheck: last_committed 30064771668 00000100:00000001:2.0:1789061120.868901:0:2406:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789061120.868903:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.868908:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.868916:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.868920:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789061120.868921:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789061120.868925:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789061120.868932:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf46c03b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540416:192.168.204.129@tcp:400:kworker.0 00000400:00000001:2.0:1789061120.868939:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789061120.868940:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789061120.868942:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895108. 00000400:00000200:2.0:1789061120.868945:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ec38 00000400:00000010:2.0:1789061120.868968:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ec38. 00000100:00000001:2.0:1789061120.868972:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.868975:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789061120.868983:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.868988:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789061120.868989:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.868991:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789061120.868992:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789061120.868993:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789061120.868995:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.868996:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789061120.868999:0:2406:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf442d05c60 time=124 v=5 (5 5 5 5) 00000100:00001000:2.0:1789061120.869002:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf442d05c18 time=103 v=5 (1 1 1 1) 00000100:00000001:2.0:1789061120.869004:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789061120.869011:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789061120.869012:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789061120.869014:0:2406:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.869016:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.869021:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.869027:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.869029:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789061120.869030:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789061120.869033:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789061120.869040:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf46c038000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2406:1875953296540288:192.168.204.129@tcp:400:kworker.0 00000400:00000001:2.0:1789061120.869046:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789061120.869047:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789061120.869048:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895478. 00000400:00000200:2.0:1789061120.869051:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ebb0 00000400:00000010:2.0:1789061120.869052:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ebb0. 00000100:00000001:2.0:1789061120.869054:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.869057:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789061120.869064:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.869070:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789061120.869071:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.869073:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789061120.869074:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789061120.869075:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789061120.869077:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061120.869078:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789061120.869083:0:2406:0:(import.c:2196:obd_at_measure()) add 60 to ffff9bf448dc6c60 time=124 v=60 (60 60 60 60) 00000100:00001000:2.0:1789061120.869086:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6c18 time=77 v=5 (1 1 1 1) 00000100:00000001:2.0:1789061120.869089:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789061120.869095:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789061120.869097:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1789061120.869101:0:2406:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1789061120.869108:0:2406:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.869111:0:2406:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789061120.869112:0:2406:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9bf4804ce000: skip recheck: last_committed 21474836545 00000100:00000001:2.0:1789061120.869114:0:2406:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789061120.869115:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061120.869120:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.869126:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061120.869128:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789061120.869130:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789061120.869133:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789061120.869141:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf46c038a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540544:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061120.869145:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.869148:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.869153:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf46c03b480 x1875953296540416/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.869160:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789061120.869164:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789061120.869168:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1fc00. 02000000:00000001:2.0:1789061120.869172:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789061120.869176:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789061120.869178:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf448dc5000 refcount=8 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000020:00000001:2.0:1789061120.869181:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789061120.869184:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf34dd7fc00. 02000000:00000001:2.0:1789061120.869190:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789061120.869192:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789061120.869199:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf46c03b480. 00000100:00000001:2.0:1789061120.869203:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789061120.869204:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.869206:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.869210:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf46c038000 x1875953296540288/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.869216:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789061120.869218:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789061120.869220:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf4472d5800. 02000000:00000001:2.0:1789061120.869223:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789061120.869224:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789061120.869226:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf442d05800 refcount=3 obd=MGC192.168.204.129@tcp 00000020:00000001:2.0:1789061120.869228:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789061120.869229:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf34dd7f700. 02000000:00000001:2.0:1789061120.869231:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789061120.869232:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789061120.869233:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf46c038000. 00000100:00000001:2.0:1789061120.869236:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789061120.869237:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.869239:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061120.869242:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf46c038a80 x1875953296540544/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061205 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061120.869249:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789061120.869251:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789061120.869252:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1ec00. 02000000:00000001:2.0:1789061120.869254:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789061120.869255:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789061120.869257:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6800 refcount=3 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000020:00000001:2.0:1789061120.869258:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789061120.869260:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf34dd7f300. 02000000:00000001:2.0:1789061120.869262:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789061120.869263:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789061120.869264:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf46c038a80. 00000100:00000001:2.0:1789061120.869266:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789061120.869267:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061120.869270:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.869277:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061120.869278:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061120.869280:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061120.869282:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789061120.870149:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.870160:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061120.870164:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.870166:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.870173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.870179:0:2398:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061120.870186:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.870190:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.870195:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.870199:0:2398:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 4 MB=0x6aa2b69db0400 00000400:00000200:2.0:1789061120.870204:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061120.870210:0:2398:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.129@tcp of length 224/224 into md 0x534815 [1] + 224 00000800:00000001:2.0:1789061120.870216:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.870230:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061120.870233:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061120.870238:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061120.870243:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061120.870251:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061120.870261:0:2398:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0F:1789061120.870350:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061120.870359:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061120.870377:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061120.870385:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061120.870388:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1789061120.873092:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061120.873099:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061120.873103:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895268. 00000400:00000200:3.0:1789061120.873111:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ee58 00000400:00000010:3.0:1789061120.873113:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ee58. 00000100:00000001:3.0:1789061120.873118:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061120.873131:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061120.873143:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061120.873150:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061120.873153:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061120.873155:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061120.873159:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061120.873162:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061120.873164:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061120.873166:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061120.873176:0:2406:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf448dc6460 time=124 v=5 (5 5 5 5) 00000100:00001000:3.0:1789061120.873180:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6418 time=112 v=5 (1 1 0 0) 00000100:00000001:3.0:1789061120.873183:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061120.873185:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061120.873187:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789061120.873191:0:2406:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061120.873195:0:2406:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789061120.873197:0:2406:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9bf4804ce000: skip recheck: last_committed 38654730077 00000100:00000001:3.0:1789061120.873198:0:2406:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789061120.873200:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061120.873205:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061120.873212:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061120.873214:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061120.873217:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061120.873220:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061120.873250:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf46c038700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540672:192.168.204.129@tcp:400:kworker.0 00000100:00000001:3.0:1789061120.873255:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061120.873258:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061120.873261:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf46c038700 x1875953296540672/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061136 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061120.873266:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061120.873268:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061120.873271:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1f600. 02000000:00000001:3.0:1789061120.873274:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061120.873277:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061120.873279:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6000 refcount=3 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000020:00000001:3.0:1789061120.873281:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061120.873284:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf34dd7fb00. 02000000:00000001:3.0:1789061120.873288:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061120.873289:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061120.873291:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf46c038700. 00000100:00000001:3.0:1789061120.873293:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061120.873295:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061120.873297:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061120.873304:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789061120.873306:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061120.873308:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061120.873311:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0F:1789061121.562087:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061121.562106:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061121.562111:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061121.562112:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061121.562138:0:244529:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061121.562141:0:244529:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061121.562144:0:244529:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061122.586335:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061122.586343:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061122.586349:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061122.586350:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061122.586375:0:237353:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061122.586378:0:237353:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061122.586381:0:237353:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061123.610044:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061123.610051:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061123.610057:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061123.610058:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789061123.610181:0:253143:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789061123.610185:0:253143:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789061123.610188:0:253143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061124.634070:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061124.634082:0:228762:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789061124.634085:0:228762:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789061124.634089:0:228762:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061124.634094:0:228762:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789061124.634097:0:228762:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061124.634099:0:228762:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061124.634102:0:228762:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061124.634107:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061124.634109:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061124.634132:0:237352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061124.634146:0:237352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061124.634149:0:237352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1789061124.698081:0:283080:0:(connection.c:75:cpu_latency_work()) work item of ffff9bf44883f220 (cpu 2) has reached its deadline 4307504860, at 4307504896 00000100:00000010:2.0:1789061124.698096:0:283080:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9bf479a68c60. 00000100:00000040:3.0:1789061124.699191:0:276965:0:(connection.c:75:cpu_latency_work()) work item of ffff9bf44883f330 (cpu 3) has reached its deadline 4307504860, at 4307504897 00000100:00000010:3.0:1789061124.699201:0:276965:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9bf479a680c0. 00010000:00000001:0.0:1789061125.658795:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061125.658805:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061125.658812:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061125.658813:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061125.658843:0:249018:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061125.658847:0:249018:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061125.658851:0:249018:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1789061125.978693:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061125.978702:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061125.978711:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836ed80. 00000020:00000040:3.0:1789061125.978716:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf442d05800 refcount=4 obd=MGC192.168.204.129@tcp 00000100:00000001:3.0:1789061125.978721:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061125.978724:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061125.978743:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061125.978750:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3500. 00000100:00000001:3.0:1789061125.978768:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.978783:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789061125.978794:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061125.978799:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.978806:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007935cbf0] to pc [ptlrpcd_01_01+1] req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.978837:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.978841:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061125.978846:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061125.978849:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836d500. 00000020:00000040:3.0:1789061125.978852:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf448dc5000 refcount=9 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000100:00000001:3.0:1789061125.978854:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061125.978855:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061125.978857:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061125.978860:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3600. 00000100:00000001:3.0:1789061125.978863:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.978868:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789061125.978875:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061125.978877:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.978881:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e28de50d] to pc [ptlrpcd_01_00+0] req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.978891:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.978894:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061125.978897:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061125.978900:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836fb80. 00000020:00000040:3.0:1789061125.978902:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6800 refcount=4 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000100:00000001:2.0:1789061125.979049:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.979053:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.979055:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061125.979061:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979069:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061125.979071:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.979075:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061125.979078:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061125.979080:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061125.979083:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2406:1875953296540800:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061125.979087:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061125.979091:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061125.979103:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.979106:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061125.979110:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1f400. 02000000:00000001:2.0:1789061125.979112:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061125.979120:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061125.979123:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895478. 00000400:00000010:2.0:1789061125.979130:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ebb0. 00000100:00000200:2.0:1789061125.979137:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296540800, portal 25 00000100:00000001:2.0:1789061125.979139:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061125.979141:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487161216 : -110001222390400 : ffff9bf45836ed80) 00000100:00000040:2.0:1789061125.979147:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979154:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061125.979158:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061125.979161:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ec38. 00000100:00000200:2.0:1789061125.979163:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875953296540800, offset 0 00000400:00000200:2.0:1789061125.979170:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061125.979178:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.979185:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061125.979190:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061125.979200:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061125.979204:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061125.979207:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061125.979210:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554648:554648:256:4294967295] 192.168.204.129@tcp LPNI seq info [554599:554599:8:4294967295] 00000400:00000200:2.0:1789061125.979215:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.979225:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061125.979231:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061125.979236:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f300. 00000800:00000200:2.0:1789061125.979243:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061125.979249:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061125.979254:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061125.979267:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061125.979271:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061125.979273:0:2406:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 2 is 4307509980 00000100:00000010:2.0:1789061125.979277:0:2406:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf479a685a0. 00000100:00000001:2.0:1789061125.979283:0:2406:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1789061125.979288:0:2406:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 3 is 4307509980 00000100:00000010:2.0:1789061125.979290:0:2406:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf479a68480. 00000100:00000001:2.0:1789061125.979293:0:2406:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1789061125.979296:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061125.979298:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979302:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979304:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.979311:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061125.979314:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789061125.979316:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.979317:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.979319:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061125.979321:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:2.0:1789061125.979323:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.979324:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061125.979326:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.979332:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.979334:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.979335:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061125.979339:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979346:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061125.979347:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.979349:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061125.979351:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061125.979353:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061125.979355:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540928:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061125.979358:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061125.979361:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061125.979362:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.979364:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061125.979367:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1ee00. 02000000:00000001:2.0:1789061125.979369:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061125.979373:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061125.979377:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895108. 00000400:00000010:2.0:1789061125.979381:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ed48. 00000100:00000200:2.0:1789061125.979384:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296540928, portal 10 00000100:00000001:2.0:1789061125.979386:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061125.979387:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487154944 : -110001222396672 : ffff9bf45836d500) 00000100:00000040:2.0:1789061125.979394:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979400:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061125.979402:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061125.979405:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e330. 00000100:00000200:2.0:1789061125.979408:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296540928, offset 0 00000400:00000200:2.0:1789061125.979411:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061125.979415:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.979419:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061125.979422:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061125.979430:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061125.979432:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061125.979434:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061125.979436:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554649:554649:255:4294967295] 192.168.204.129@tcp LPNI seq info [554600:554600:7:4294967295] 00000400:00000200:2.0:1789061125.979440:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.979447:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061125.979451:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061125.979454:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f700. 00000800:00000200:2.0:1789061125.979457:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061125.979461:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061125.979463:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061125.979466:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061125.979471:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: using a timeout of 29475 usecs (30 jiffies) 00000400:00000200:2.0:1789061125.979474:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061125.979476:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307506207 00000100:00000001:2.0:1789061125.979479:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307509980 00000100:00000001:2.0:1789061125.979481:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307506207 00000100:00000001:2.0:1789061125.979483:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307509980 00000100:00000001:2.0:1789061125.979485:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061125.979487:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979489:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979491:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979501:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.979503:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789061125.979538:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061125.979542:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f300. 00000400:00000200:2.0:1789061125.979546:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061125.979551:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061125.979557:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ec38 00000400:00000010:2.0:1789061125.979559:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ec38. 00000100:00000001:2.0:1789061125.979563:0:2397:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061125.979572:0:2397:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979581:0:2397:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061125.979584:0:2397:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979590:0:2397:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979592:0:2397:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789061125.979800:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061125.979802:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f700. 00000400:00000200:2.0:1789061125.979806:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061125.979809:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061125.979812:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e330 00000400:00000010:2.0:1789061125.979814:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e330. 00000100:00000001:2.0:1789061125.979817:0:2397:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061125.979820:0:2397:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979828:0:2397:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061125.979830:0:2397:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.979835:0:2397:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.979837:0:2397:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1789061125.980141:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061125.980142:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061125.980145:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061125.980151:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3100. 00000100:00000001:3.0:1789061125.980155:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.980162:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789061125.980170:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061125.980174:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.980178:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d018342c] to pc [ptlrpcd_01_01+1] req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.980202:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.980205:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061125.980209:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061125.980213:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836d180. 00000020:00000040:3.0:1789061125.980216:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6000 refcount=4 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000100:00000001:3.0:1789061125.980218:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061125.980219:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061125.980221:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061125.980224:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3e00. 00000100:00000001:3.0:1789061125.980226:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.980231:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789061125.980238:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061125.980240:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.980245:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e43e7462] to pc [ptlrpcd_01_00+0] req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.980253:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.980254:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1789061125.980258:0:250456:0:(pinger.c:247:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1789061125.980268:0:250456:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12843) 00000100:00000001:2.0:1789061125.980373:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.980376:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.980378:0:2405:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061125.980383:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.980390:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061125.980392:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.980394:0:2405:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061125.980396:0:2405:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061125.980399:0:2405:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061125.980402:0:2405:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541184:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061125.980412:0:2405:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061125.980416:0:2405:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061125.980417:0:2405:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.980419:0:2405:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061125.980423:0:2405:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1e000. 02000000:00000001:2.0:1789061125.980425:0:2405:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061125.980430:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061125.980433:0:2405:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895318. 00000400:00000010:2.0:1789061125.980437:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e330. 00000100:00000200:2.0:1789061125.980442:0:2405:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541184, portal 4 00000100:00000001:2.0:1789061125.980444:0:2405:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061125.980445:0:2405:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487154048 : -110001222397568 : ffff9bf45836d180) 00000100:00000040:2.0:1789061125.980451:0:2405:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.980458:0:2405:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061125.980461:0:2405:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061125.980463:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ec38. 00000100:00000200:2.0:1789061125.980465:0:2405:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875953296541184, offset 0 00000400:00000200:2.0:1789061125.980470:0:2405:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061125.980475:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.980480:0:2405:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061125.980483:0:2405:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061125.980487:0:2405:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061125.980489:0:2405:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061125.980492:0:2405:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061125.980494:0:2405:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554650:554650:256:4294967295] 192.168.204.129@tcp LPNI seq info [554601:554601:8:4294967295] 00000400:00000200:2.0:1789061125.980498:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.980504:0:2405:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061125.980509:0:2405:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061125.980513:0:2405:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f700. 00000800:00000200:2.0:1789061125.980517:0:2405:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061125.980527:0:2405:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061125.980530:0:2405:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061125.982023:0:2405:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061125.982033:0:2405:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: using a timeout of 111147 usecs (112 jiffies) 00000400:00000200:2.0:1789061125.982040:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061125.982044:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307506292 00000100:00000001:2.0:1789061125.982048:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307509980 00000100:00000001:2.0:1789061125.982050:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307506292 00000100:00000001:2.0:1789061125.982052:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307509980 00000100:00000001:2.0:1789061125.982054:0:2405:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061125.982057:0:2405:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.982061:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.982064:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.982073:0:2405:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061125.982075:0:2405:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789061125.982078:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.982079:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.982080:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061125.982083:0:2405:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1789061125.982085:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.982086:0:2405:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061125.982088:0:2405:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.982089:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.982090:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.982091:0:2405:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061125.982102:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.982109:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061125.982111:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.982114:0:2405:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061125.982117:0:2405:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061125.982119:0:2405:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061125.982123:0:2405:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541056:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061125.982144:0:2405:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061125.982150:0:2405:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061125.982152:0:2405:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.982154:0:2405:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061125.982161:0:2405:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf442c1e600. 02000000:00000001:2.0:1789061125.982162:0:2405:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061125.982167:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061125.982171:0:2405:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895268. 00000400:00000010:2.0:1789061125.982177:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e990. 00000100:00000200:2.0:1789061125.982183:0:2405:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541056, portal 10 00000100:00000001:2.0:1789061125.982186:0:2405:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061125.982187:0:2405:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487164800 : -110001222386816 : ffff9bf45836fb80) 00000100:00000040:2.0:1789061125.982194:0:2405:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.982202:0:2405:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061125.982205:0:2405:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061125.982207:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e440. 00000100:00000200:2.0:1789061125.982209:0:2405:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296541056, offset 0 00000400:00000200:2.0:1789061125.982216:0:2405:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061125.982223:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.982229:0:2405:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061125.982234:0:2405:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061125.982237:0:2405:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061125.982240:0:2405:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061125.982243:0:2405:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061125.982246:0:2405:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554651:554651:255:4294967295] 192.168.204.129@tcp LPNI seq info [554602:554602:7:4294967295] 00000400:00000200:2.0:1789061125.982251:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.982259:0:2405:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061125.982265:0:2405:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061125.982270:0:2405:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf34dd7f300. 00000800:00000200:2.0:1789061125.982275:0:2405:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061125.982286:0:2405:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061125.982290:0:2405:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf34dd7f300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061125.982293:0:2405:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061125.982296:0:2405:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: using a timeout of 22406 usecs (23 jiffies) 00000400:00000200:2.0:1789061125.982299:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061125.982301:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307506203 00000100:00000001:2.0:1789061125.982304:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307509980 00000100:00000001:2.0:1789061125.982305:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307506203 00000100:00000001:2.0:1789061125.982307:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307509980 00000100:00000001:2.0:1789061125.982309:0:2405:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061125.982311:0:2405:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.982313:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.982315:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.982334:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.982336:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061125.982338:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.982340:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1789061125.982448:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789061125.982452:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f700. 00000400:00000200:3.0:1789061125.982460:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789061125.982464:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:3.0:1789061125.982469:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ec38 00000400:00000010:3.0:1789061125.982472:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ec38. 00000100:00000001:3.0:1789061125.982475:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789061125.982479:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.982488:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061125.982491:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.982496:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.982498:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:3.0:1789061125.982559:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1789061125.982562:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf34dd7f300. 00000400:00000200:3.0:1789061125.982564:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789061125.982567:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:3.0:1789061125.982569:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e440 00000400:00000010:3.0:1789061125.982571:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e440. 00000100:00000001:3.0:1789061125.982573:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:3.0:1789061125.982576:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.982582:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061125.982584:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.982590:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.982592:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1789061125.983134:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.983145:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061125.983148:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.983150:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061125.983156:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.983160:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061125.983166:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.983169:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.983172:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.983176:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 25 MB=0x6aa2b69db0480 00000400:00000200:2.0:1789061125.983180:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.983187:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.129@tcp of length 224/224 into md 0x534825 [1] + 224 00000800:00000001:2.0:1789061125.983192:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.983197:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061125.983200:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061125.983211:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061125.983216:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061125.983221:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061125.983229:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.983249:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061125.983255:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.983258:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.984977:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061125.984985:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789061125.984996:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061125.985012:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061125.985016:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895478. 00000400:00000200:3.0:1789061125.985023:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ebb0 00000400:00000010:3.0:1789061125.985026:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ebb0. 00000100:00000001:3.0:1789061125.985032:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061125.985044:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061125.985054:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.985062:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061125.985064:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.985066:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061125.985070:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061125.985072:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061125.985074:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061125.985078:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061125.985084:0:2406:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf442d05c60 time=129 v=5 (5 5 5 5) 00000100:00001000:3.0:1789061125.985093:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf442d05c18 time=108 v=5 (1 1 1 1) 00000100:00000001:3.0:1789061125.985107:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061125.985108:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061125.985111:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789061125.985113:0:2406:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.985117:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.985123:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.985129:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061125.985132:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061125.985134:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061125.985138:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061125.985145:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2406:1875953296540800:192.168.204.129@tcp:400:kworker.0 00000100:00000001:3.0:1789061125.985150:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.985158:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061125.985162:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836ed80 x1875953296540800/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.985169:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061125.985171:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061125.985174:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1f400. 02000000:00000001:3.0:1789061125.985177:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061125.985184:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061125.985187:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf442d05800 refcount=3 obd=MGC192.168.204.129@tcp 00000020:00000001:3.0:1789061125.985189:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061125.985192:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3500. 02000000:00000001:3.0:1789061125.985198:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061125.985200:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061125.985202:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836ed80. 00000100:00000001:3.0:1789061125.985206:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061125.985207:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.985210:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.985212:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061125.985213:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789061125.985215:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.985216:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1789061125.991165:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061125.991186:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789061125.991191:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061125.991193:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789061125.991202:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061125.991210:0:2398:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:3.0:1789061125.991219:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061125.991224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061125.991230:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061125.991234:0:2398:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0500 00000400:00000200:3.0:1789061125.991239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061125.991246:0:2398:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x534835 [1] + 224 00000800:00000001:3.0:1789061125.991252:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061125.991297:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789061125.991301:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789061125.991317:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:3.0:1789061125.991325:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061125.991334:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789061125.991345:0:2398:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.991423:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.991428:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789061125.991448:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789061125.991459:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061125.991463:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1789061125.994042:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789061125.994050:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789061125.994055:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895108. 00000400:00000200:2.0:1789061125.994064:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ed48 00000400:00000010:2.0:1789061125.994066:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ed48. 00000100:00000001:2.0:1789061125.994071:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061125.994083:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789061125.994095:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.994104:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789061125.994106:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.994109:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789061125.994113:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789061125.994116:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789061125.994118:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061125.994121:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789061125.994133:0:2406:0:(import.c:2196:obd_at_measure()) add 60 to ffff9bf448dc5460 time=129 v=60 (60 60 60 60) 00000100:00001000:2.0:1789061125.994138:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc5418 time=106 v=5 (1 1 1 1) 00000100:00000001:2.0:1789061125.994141:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789061125.994143:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789061125.994147:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1789061125.994152:0:2406:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.994176:0:2406:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789061125.994179:0:2406:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9bf4804ce000: skip recheck: last_committed 30064771668 00000100:00000001:2.0:1789061125.994181:0:2406:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789061125.994182:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061125.994188:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.994195:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061125.994198:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789061125.994200:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789061125.994204:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789061125.994212:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296540928:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061125.994217:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.994220:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061125.994223:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836d500 x1875953296540928/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.994230:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789061125.994233:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789061125.994237:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1ee00. 02000000:00000001:2.0:1789061125.994240:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789061125.994243:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789061125.994245:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf448dc5000 refcount=8 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000020:00000001:2.0:1789061125.994248:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789061125.994251:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3600. 02000000:00000001:2.0:1789061125.994258:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789061125.994260:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789061125.994262:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836d500. 00000100:00000001:2.0:1789061125.994266:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789061125.994267:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.994270:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061125.994280:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061125.994282:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061125.994290:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061125.994293:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789061125.996189:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996205:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061125.996209:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996212:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061125.996220:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996228:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061125.996238:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996244:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996250:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996255:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0580 00000400:00000200:2.0:1789061125.996260:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996266:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x534855 [1] + 224 00000800:00000001:2.0:1789061125.996273:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996276:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061125.996280:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061125.996285:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061125.996292:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061125.996300:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061125.996310:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.996323:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061125.996331:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996335:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061125.996336:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996338:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061125.996341:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996344:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061125.996348:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996350:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996353:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996365:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 4 MB=0x6aa2b69db0600 00000400:00000200:2.0:1789061125.996369:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061125.996373:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.129@tcp of length 224/224 into md 0x534845 [1] + 224 00000800:00000001:2.0:1789061125.996377:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996384:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061125.996386:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061125.996389:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061125.996392:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061125.996396:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061125.996405:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061125.996412:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061125.996415:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061125.996418:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996472:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061125.996476:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789061125.996482:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061125.996484:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061125.996487:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895268. 00000400:00000200:3.0:1789061125.996491:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e990 00000400:00000010:3.0:1789061125.996494:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e990. 00000100:00000001:3.0:1789061125.996496:0:2405:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061125.996501:0:2405:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061125.996510:0:2405:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.996516:0:2405:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061125.996518:0:2405:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996521:0:2405:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061125.996524:0:2405:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061125.996527:0:2405:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061125.996529:0:2405:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061125.996532:0:2405:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061125.996542:0:2405:0:(import.c:2196:obd_at_measure()) add 60 to ffff9bf448dc6c60 time=129 v=60 (60 60 60 60) 00000100:00001000:3.0:1789061125.996546:0:2405:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6c18 time=82 v=5 (1 1 1 1) 00000100:00000001:3.0:1789061125.996549:0:2405:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061125.996551:0:2405:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061125.996554:0:2405:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1789061125.996565:0:2405:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1789061125.996573:0:2405:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.996577:0:2405:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789061125.996579:0:2405:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9bf4804ce000: skip recheck: last_committed 21474836545 00000100:00000001:3.0:1789061125.996581:0:2405:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789061125.996583:0:2405:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.996587:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.996593:0:2405:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061125.996596:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061125.996598:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061125.996602:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061125.996610:0:2405:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541056:192.168.204.129@tcp:400:kworker.0 00000400:00000001:3.0:1789061125.996616:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061125.996617:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061125.996619:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895318. 00000400:00000200:3.0:1789061125.996622:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e330 00000400:00000010:3.0:1789061125.996624:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e330. 00000100:00000001:3.0:1789061125.996626:0:2405:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061125.996629:0:2405:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061125.996637:0:2405:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.996654:0:2405:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061125.996655:0:2405:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996657:0:2405:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061125.996658:0:2405:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061125.996659:0:2405:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061125.996666:0:2405:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061125.996668:0:2405:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061125.996673:0:2405:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf448dc6460 time=129 v=5 (5 5 5 5) 00000100:00001000:3.0:1789061125.996677:0:2405:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6418 time=117 v=5 (1 1 0 0) 00000100:00000001:3.0:1789061125.996679:0:2405:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061125.996680:0:2405:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061125.996682:0:2405:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789061125.996684:0:2405:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.996686:0:2405:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789061125.996688:0:2405:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9bf4804ce000: skip recheck: last_committed 38654730077 00000100:00000001:3.0:1789061125.996689:0:2405:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789061125.996690:0:2405:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061125.996694:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.996699:0:2405:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061125.996701:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061125.996702:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061125.996705:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061125.996711:0:2405:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541184:192.168.204.129@tcp:400:kworker.0 00000100:00000001:3.0:1789061125.996715:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996718:0:2405:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061125.996720:0:2405:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836fb80 x1875953296541056/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061210 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.996726:0:2405:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061125.996728:0:2405:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061125.996732:0:2405:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1e600. 02000000:00000001:3.0:1789061125.996735:0:2405:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061125.996737:0:2405:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061125.996739:0:2405:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6800 refcount=3 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000020:00000001:3.0:1789061125.996741:0:2405:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061125.996744:0:2405:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3100. 02000000:00000001:3.0:1789061125.996755:0:2405:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061125.996757:0:2405:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061125.996759:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836fb80. 00000100:00000001:3.0:1789061125.996762:0:2405:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061125.996763:0:2405:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996765:0:2405:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061125.996769:0:2405:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836d180 x1875953296541184/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061141 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061125.996775:0:2405:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061125.996776:0:2405:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061125.996778:0:2405:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf442c1e000. 02000000:00000001:3.0:1789061125.996780:0:2405:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061125.996781:0:2405:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061125.996782:0:2405:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6000 refcount=3 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000020:00000001:3.0:1789061125.996784:0:2405:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061125.996785:0:2405:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3e00. 02000000:00000001:3.0:1789061125.996788:0:2405:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061125.996789:0:2405:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061125.996790:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836d180. 00000100:00000001:3.0:1789061125.996792:0:2405:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061125.996793:0:2405:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996796:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061125.996802:0:2405:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789061125.996803:0:2405:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061125.996805:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061125.996808:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061126.682162:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061126.682201:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061126.682209:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061126.682210:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061126.682238:0:244529:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061126.682241:0:244529:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061126.682245:0:244529:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061127.707316:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061127.707324:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061127.707329:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061127.707331:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061127.707356:0:237353:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061127.707358:0:237353:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061127.707361:0:237353:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061128.730408:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061128.730416:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061128.730422:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061128.730424:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789061128.732742:0:253143:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789061128.732747:0:253143:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789061128.732752:0:253143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1789061129.242059:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf46197b000] -> 12345-192.168.204.29@tcp (6) 00000800:00000200:2.0:1789061129.242070:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.29@tcp ip 192.168.204.29:988 00000800:00000200:2.0:1789061129.242076:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf44640ae00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:3.0:1789061129.245910:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1789061129.245924:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061129.245935:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789061129.245940:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061129.245942:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:1.0F:1789061129.475746:0:286002:0:(llite_lib.c:4258:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061129.754068:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061129.754074:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061129.754082:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061129.754084:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061129.754109:0:237352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061129.754112:0:237352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061129.754116:0:237352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1789061129.818137:0:273614:0:(connection.c:75:cpu_latency_work()) work item of ffff9bf44883f220 (cpu 2) has reached its deadline 4307509980, at 4307510016 00000100:00000010:2.0:1789061129.818155:0:273614:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9bf479a685a0. 00000100:00000040:3.0:1789061129.818311:0:276965:0:(connection.c:75:cpu_latency_work()) work item of ffff9bf44883f330 (cpu 3) has reached its deadline 4307509980, at 4307510016 00000100:00000010:3.0:1789061129.818317:0:276965:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9bf479a68480. 00010000:00000001:0.0:1789061130.778129:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061130.778141:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061130.778149:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061130.778154:0:228762:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789061130.778156:0:228762:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789061130.778164:0:228762:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1789061130.778166:0:228762:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1789061130.778168:0:228762:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061130.778171:0:228762:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789061130.778173:0:228762:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061130.778175:0:228762:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061130.778177:0:228762:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061130.778212:0:249018:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061130.778216:0:249018:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061130.778245:0:249018:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1789061131.100643:0:285885:0:(pinger.c:229:ptlrpc_pinger_process_import()) f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061131.100655:0:285885:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061131.100665:0:285885:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836d180. 00000020:00000040:3.0:1789061131.100688:0:285885:0:(genops.c:1209:class_import_get()) import ffff9bf442d05800 refcount=4 obd=MGC192.168.204.129@tcp 00000100:00000001:3.0:1789061131.100693:0:285885:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061131.100696:0:285885:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061131.100701:0:285885:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061131.100709:0:285885:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3c00. 00000100:00000001:3.0:1789061131.100717:0:285885:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.100730:0:285885:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789061131.100740:0:285885:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061131.100746:0:285885:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.100752:0:285885:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e43e7462] to pc [ptlrpcd_01_01+1] req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.100785:0:285885:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.100787:0:285885:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061131.100791:0:285885:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061131.100794:0:285885:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836fb80. 00000020:00000040:3.0:1789061131.100796:0:285885:0:(genops.c:1209:class_import_get()) import ffff9bf448dc5000 refcount=9 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000100:00000001:3.0:1789061131.100799:0:285885:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061131.100800:0:285885:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061131.100802:0:285885:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061131.100806:0:285885:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3e00. 00000100:00000001:3.0:1789061131.100809:0:285885:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.100814:0:285885:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789061131.100821:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061131.100822:0:285885:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061131.100827:0:285885:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.100828:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1789061131.100832:0:285885:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d018342c] to pc [ptlrpcd_01_00+0] req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.100833:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061131.100844:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.100846:0:285885:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.100850:0:285885:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1789061131.100860:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061131.100864:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.100866:0:285885:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:2.0:1789061131.100871:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:3.0:1789061131.100872:0:285885:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836ed80. 00000020:00000040:3.0:1789061131.100875:0:285885:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6800 refcount=4 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000100:00000001:3.0:1789061131.100877:0:285885:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061131.100878:0:285885:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061131.100881:0:285885:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061131.100884:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1789061131.100888:0:285885:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3100. 02000000:00000001:2.0:1789061131.100889:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.100893:0:285885:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061131.100899:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2406:1875953296541312:192.168.204.129@tcp:400:kworker.0 00000100:00000040:3.0:1789061131.100903:0:285885:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1789061131.100907:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1789061131.100914:0:285885:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1789061131.100915:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1789061131.100918:0:285885:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061131.100919:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061131.100924:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1789061131.100925:0:285885:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007935cbf0] to pc [ptlrpcd_01_01+1] req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1789061131.100930:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf4586b8c00. 02000000:00000001:2.0:1789061131.100935:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.100942:0:285885:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.100945:0:285885:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1789061131.100988:0:285885:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1789061131.100993:0:285885:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf45836d500. 00000020:00000040:3.0:1789061131.100995:0:285885:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6000 refcount=4 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000100:00000001:3.0:1789061131.100997:0:285885:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1789061131.100999:0:285885:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1789061131.101000:0:285885:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1789061131.101003:0:285885:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf4816f3500. 00000100:00000001:3.0:1789061131.101006:0:285885:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.101010:0:285885:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1789061131.101018:0:285885:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1789061131.101020:0:285885:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.101024:0:285885:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e28de50d] to pc [ptlrpcd_01_00+0] req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.101042:0:285885:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.101044:0:285885:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1789061131.101047:0:285885:0:(pinger.c:247:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1789061131.101052:0:285885:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12849) 00000400:00000200:2.0:1789061131.101066:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:3.0:1789061131.101069:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061131.101073:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1789061131.101073:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf461895108. 00000100:00000001:3.0:1789061131.101075:0:2405:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1789061131.101082:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1789061131.101084:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007ed48. 00000100:00000200:2.0:1789061131.101094:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541312, portal 25 00000100:00000001:2.0:1789061131.101098:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061131.101099:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487154048 : -110001222397568 : ffff9bf45836d180) 00000100:00000040:2.0:1789061131.101108:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.101115:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061131.101118:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061131.101121:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e5d8. 00000100:00000200:2.0:1789061131.101124:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875953296541312, offset 0 00000400:00000200:2.0:1789061131.101131:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061131.101141:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.101149:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061131.101155:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061131.101159:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061131.101163:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061131.101165:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061131.101169:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554652:554652:256:4294967295] 192.168.204.129@tcp LPNI seq info [554603:554603:8:4294967295] 00000400:00000200:2.0:1789061131.101174:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.101183:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061131.101189:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061131.101201:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf459212400. 00000800:00000200:2.0:1789061131.101214:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061131.101222:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061131.101227:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf459212400 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061131.101241:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061131.101246:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061131.101249:0:2406:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 2 is 4307515102 00000100:00000010:2.0:1789061131.101254:0:2406:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf479a68a20. 00000100:00000001:2.0:1789061131.101261:0:2406:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1789061131.101265:0:2406:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 3 is 4307515102 00000100:00000010:2.0:1789061131.101268:0:2406:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf479a685a0. 00000100:00000001:2.0:1789061131.101271:0:2406:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1789061131.101274:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061131.101278:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.101281:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.101283:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061131.101292:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061131.101295:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789061131.101296:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061131.101298:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061131.101299:0:2406:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1789061131.101303:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.101309:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1789061131.101310:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061131.101312:0:2406:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1789061131.101314:0:2406:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1789061131.101315:0:2406:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1789061131.101318:0:2406:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296541568:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061131.101320:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1789061131.101323:0:2406:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1789061131.101324:0:2406:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061131.101332:0:2406:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1789061131.101334:0:2406:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf4586b9c00. 02000000:00000001:2.0:1789061131.101336:0:2406:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061131.101340:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:2.0:1789061131.101343:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf4618956e0. 00000400:00000010:2.0:1789061131.101347:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e550. 00000100:00000200:2.0:1789061131.101350:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541568, portal 10 00000100:00000001:2.0:1789061131.101351:0:2406:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1789061131.101352:0:2406:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487161216 : -110001222390400 : ffff9bf45836ed80) 00000100:00000040:2.0:1789061131.101359:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.101365:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1789061131.101367:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:2.0:1789061131.101369:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf46007e880. 00000100:00000200:2.0:1789061131.101371:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296541568, offset 0 00000400:00000200:2.0:1789061131.101375:0:2406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061131.101378:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.101382:0:2406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061131.101385:0:2406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1789061131.101387:0:2406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:2.0:1789061131.101390:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1789061131.101392:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:2.0:1789061131.101395:0:2406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554653:554653:255:4294967295] 192.168.204.129@tcp LPNI seq info [554604:554604:7:4294967295] 00000400:00000200:2.0:1789061131.101399:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.101405:0:2406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:2.0:1789061131.101409:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:2.0:1789061131.101413:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf459212d00. 00000800:00000200:2.0:1789061131.101416:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:2.0:1789061131.101421:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:2.0:1789061131.101423:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf459212d00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061131.101426:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061131.101436:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: using a timeout of 22406 usecs (23 jiffies) 00000400:00000200:2.0:1789061131.101440:0:2406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:2.0:1789061131.101442:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307511322 00000100:00000001:2.0:1789061131.101445:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307515102 00000100:00000001:2.0:1789061131.101447:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307511322 00000100:00000001:2.0:1789061131.101449:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307515102 00000100:00000001:2.0:1789061131.101451:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061131.101453:0:2406:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.101456:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.101458:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061131.101461:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061131.101462:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1789061131.101464:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061131.101465:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061131.101467:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.101468:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789061131.101879:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061131.101884:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf459212400. 00000400:00000200:2.0:1789061131.101891:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061131.101897:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061131.101904:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e5d8 00000400:00000010:2.0:1789061131.101908:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e5d8. 00000100:00000001:2.0:1789061131.101913:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061131.101919:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.101929:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061131.101932:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.101940:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.101942:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1789061131.102072:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1789061131.102074:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061131.102078:0:2405:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1789061131.102080:0:2405:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789061131.102083:0:2405:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1789061131.102087:0:2405:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541440:192.168.204.129@tcp:400:kworker.0 00000100:00000001:3.0:1789061131.102090:0:2405:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1789061131.102104:0:2405:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1789061131.102106:0:2405:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061131.102108:0:2405:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1789061131.102112:0:2405:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf451899a00. 02000000:00000001:3.0:1789061131.102114:0:2405:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789061131.102119:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:3.0:1789061131.102123:0:2405:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf450049c60. 00000400:00000010:3.0:1789061131.102127:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf442c1a7f8. 00000100:00000200:3.0:1789061131.102131:0:2405:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541440, portal 10 00000100:00000001:3.0:1789061131.102133:0:2405:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1789061131.102134:0:2405:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487164800 : -110001222386816 : ffff9bf45836fb80) 00000100:00000040:3.0:1789061131.102142:0:2405:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.102149:0:2405:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789061131.102159:0:2405:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:3.0:1789061131.102162:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf442c1add0. 00000800:00000200:2.0:1789061131.102165:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000200:3.0:1789061131.102166:0:2405:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296541440, offset 0 00000800:00000010:2.0:1789061131.102171:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf459212d00. 00000400:00000200:3.0:1789061131.102176:0:2405:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061131.102178:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789061131.102181:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061131.102187:0:2405:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:2.0:1789061131.102193:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:3.0:1789061131.102194:0:2405:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1789061131.102201:0:2405:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:3.0:1789061131.102205:0:2405:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1789061131.102207:0:2405:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:3.0:1789061131.102211:0:2405:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554654:554654:255:4294967295] 192.168.204.129@tcp LPNI seq info [554605:554605:7:4294967295] 00000400:00000200:3.0:1789061131.102215:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061131.102223:0:2405:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:3.0:1789061131.102229:0:2405:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000400:00000200:2.0:1789061131.102231:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e880 00000800:00000010:3.0:1789061131.102234:0:2405:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf4816f3900. 00000400:00000010:2.0:1789061131.102235:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e880. 00000800:00000200:3.0:1789061131.102241:0:2405:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000100:00000001:2.0:1789061131.102241:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061131.102248:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1789061131.102250:0:2405:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:3.0:1789061131.102255:0:2405:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf4816f3900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1789061131.102264:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1789061131.102267:0:2405:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061131.102270:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789061131.102273:0:2405:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: using a timeout of 29471 usecs (30 jiffies) 00000400:00000200:3.0:1789061131.102280:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:3.0:1789061131.102285:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307511330 00000100:00000001:2.0:1789061131.102286:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.102291:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307515102 00000100:00000001:2.0:1789061131.102291:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1789061131.102295:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307511330 00000100:00000001:3.0:1789061131.102297:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307515102 00000100:00000001:3.0:1789061131.102299:0:2405:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061131.102301:0:2405:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.102304:0:2405:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000800:00000200:2.0:1789061131.102306:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:3.0:1789061131.102310:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1789061131.102311:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf4816f3900. 00000400:00000200:2.0:1789061131.102317:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061131.102324:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061131.102327:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf442c1add0 00000400:00000010:2.0:1789061131.102331:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf442c1add0. 00000100:00000001:2.0:1789061131.102336:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1789061131.102340:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1789061131.102343:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1789061131.102344:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1789061131.102348:0:2405:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1789061131.102351:0:2405:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1789061131.102355:0:2405:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.102358:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1789061131.102362:0:2405:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf45836d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541696:192.168.204.129@tcp:400:kworker.0 00000100:00000040:2.0:1789061131.102366:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.102367:0:2405:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1789061131.102372:0:2405:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1789061131.102374:0:2405:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.102377:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061131.102378:0:2405:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1789061131.102382:0:2398:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000010:3.0:1789061131.102383:0:2405:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf451899200. 02000000:00000001:3.0:1789061131.102387:0:2405:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1789061131.102391:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:3.0:1789061131.102394:0:2405:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf4500496e0. 00000400:00000010:3.0:1789061131.102398:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf442c1a990. 00000100:00000200:3.0:1789061131.102400:0:2405:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541696, portal 4 00000100:00000001:3.0:1789061131.102402:0:2405:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1789061131.102403:0:2405:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634072487154944 : -110001222396672 : ffff9bf45836d500) 00000100:00000040:3.0:1789061131.102410:0:2405:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.102416:0:2405:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1789061131.102418:0:2405:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:3.0:1789061131.102420:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf442c1a660. 00000100:00000200:3.0:1789061131.102422:0:2405:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875953296541696, offset 0 00000400:00000200:3.0:1789061131.102426:0:2405:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:3.0:1789061131.102430:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061131.102433:0:2405:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:3.0:1789061131.102435:0:2405:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1789061131.102437:0:2405:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:3.0:1789061131.102439:0:2405:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1789061131.102441:0:2405:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:3.0:1789061131.102449:0:2405:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554655:554655:255:4294967295] 192.168.204.129@tcp LPNI seq info [554606:554606:7:4294967295] 00000400:00000200:3.0:1789061131.102453:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061131.102458:0:2405:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:3.0:1789061131.102462:0:2405:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:3.0:1789061131.102465:0:2405:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf4816f3f00. 00000800:00000200:3.0:1789061131.102470:0:2405:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:3.0:1789061131.102475:0:2405:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:3.0:1789061131.102478:0:2405:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf4816f3f00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1789061131.102488:0:2405:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.102492:0:2405:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: using a timeout of 111088 usecs (112 jiffies) 00000400:00000200:3.0:1789061131.102495:0:2405:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:3.0:1789061131.102497:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307511412 00000100:00000001:3.0:1789061131.102499:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307515102 00000100:00000001:3.0:1789061131.102502:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307511412 00000100:00000001:3.0:1789061131.102504:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307515102 00000100:00000001:3.0:1789061131.102506:0:2405:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061131.102508:0:2405:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.102511:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.102514:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1789061131.102519:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1789061131.102520:0:2405:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789061131.102523:0:2405:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000010:2.0:1789061131.102523:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf4816f3f00. 00000100:00000001:3.0:1789061131.102529:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1789061131.102530:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1789061131.102531:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1789061131.102535:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061131.102538:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:3.0:1789061131.102540:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1789061131.102556:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf442c1a660 00000400:00000010:2.0:1789061131.102558:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf442c1a660. 00000100:00000001:2.0:1789061131.102562:0:2397:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061131.102643:0:2397:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.102651:0:2397:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061131.102654:0:2397:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.102661:0:2397:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.102663:0:2397:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1789061131.108365:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.109688:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061131.109696:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.109702:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.109713:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.109722:0:2398:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061131.110668:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.110676:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.110682:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.110689:0:2398:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 25 MB=0x6aa2b69db0680 00000400:00000200:2.0:1789061131.110694:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.110700:0:2398:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.129@tcp of length 224/224 into md 0x534865 [1] + 224 00000800:00000001:2.0:1789061131.110710:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.110719:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.110724:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061131.110729:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061131.111042:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061131.111055:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061131.111067:0:2398:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.111136:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061131.111142:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1789061131.111762:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061131.111777:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.111786:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061131.111789:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.111812:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.111819:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.111825:0:2398:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061131.111832:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.111835:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.111838:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.111842:0:2398:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0780 00000400:00000200:2.0:1789061131.111846:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.111851:0:2398:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x534875 [1] + 224 00000800:00000001:2.0:1789061131.111856:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.111865:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.111867:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061131.111871:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061131.111874:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061131.111881:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061131.111890:0:2398:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.111897:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061131.111901:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.111904:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1789061131.114358:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061131.114367:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061131.114371:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf461895108. 00000400:00000200:3.0:1789061131.114381:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007ed48 00000400:00000010:3.0:1789061131.114384:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007ed48. 00000100:00000001:3.0:1789061131.114410:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061131.114423:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061131.114434:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.114441:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061131.114443:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.114447:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061131.114451:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061131.114453:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061131.114455:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061131.114458:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061131.114464:0:2406:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf442d05c60 time=135 v=5 (5 5 5 5) 00000100:00001000:3.0:1789061131.114468:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf442d05c18 time=114 v=5 (1 1 1 1) 00000100:00000001:3.0:1789061131.114471:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061131.114472:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061131.114475:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789061131.114477:0:2406:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.114480:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.116642:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.116675:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061131.116686:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061131.116688:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061131.116693:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061131.116702:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2406:1875953296541312:192.168.204.129@tcp:400:kworker.0 00000100:00000001:3.0:1789061131.116718:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.116722:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061131.116725:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836d180 x1875953296541312/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.116741:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061131.116778:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061131.116796:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf4586b8c00. 02000000:00000001:3.0:1789061131.116802:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061131.116805:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061131.116806:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf442d05800 refcount=3 obd=MGC192.168.204.129@tcp 00000020:00000001:3.0:1789061131.116809:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061131.116812:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3c00. 02000000:00000001:3.0:1789061131.116815:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061131.116817:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061131.116819:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836d180. 00000100:00000001:3.0:1789061131.116823:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061131.116825:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.116829:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.116832:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061131.116834:0:2406:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000800:00000001:2.0:1789061131.117113:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117130:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061131.117133:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117136:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.117143:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117149:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061131.117158:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117163:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117169:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117173:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 4 MB=0x6aa2b69db0800 00000400:00000200:2.0:1789061131.117178:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117191:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.129@tcp of length 224/224 into md 0x534895 [1] + 224 00000800:00000001:2.0:1789061131.117200:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117205:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.117209:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061131.117213:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061131.117221:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061131.117227:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061131.117237:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.117258:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061131.117275:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117279:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061131.117281:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117283:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.117286:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117289:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061131.117293:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117296:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117299:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117302:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0700 00000400:00000200:2.0:1789061131.117305:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061131.117310:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x534885 [1] + 224 00000800:00000001:2.0:1789061131.117314:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117379:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061131.117382:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061131.117385:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061131.117389:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061131.117394:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061131.117403:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.117417:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061131.117422:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061131.117426:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1789061131.117439:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1789061131.117441:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1789061131.117445:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf4618956e0. 00000400:00000200:2.0:1789061131.117449:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf46007e550 00000400:00000010:2.0:1789061131.117452:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf46007e550. 00000100:00000001:2.0:1789061131.117455:0:2406:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061131.117459:0:2406:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1789061131.117468:0:2406:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.117474:0:2406:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1789061131.117476:0:2406:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061131.117478:0:2406:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1789061131.117481:0:2406:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1789061131.117483:0:2406:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1789061131.117486:0:2406:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1789061131.117488:0:2406:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1789061131.117500:0:2406:0:(import.c:2196:obd_at_measure()) add 60 to ffff9bf448dc6c60 time=135 v=60 (60 60 60 60) 00000100:00001000:2.0:1789061131.117504:0:2406:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6c18 time=88 v=5 (1 1 1 1) 00000100:00000001:2.0:1789061131.117507:0:2406:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1789061131.117509:0:2406:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1789061131.117512:0:2406:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1789061131.117516:0:2406:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1789061131.117522:0:2406:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.117526:0:2406:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1789061131.117528:0:2406:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9bf4804ce000: skip recheck: last_committed 21474836545 00000100:00000001:2.0:1789061131.117529:0:2406:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1789061131.117531:0:2406:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1789061131.117555:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.117563:0:2406:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1789061131.117565:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1789061131.117567:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1789061131.117571:0:2406:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789061131.117579:0:2406:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f9405128-a312-44b7-8d65-caa32bc1b7dc:2406:1875953296541568:192.168.204.129@tcp:400:kworker.0 00000100:00000001:2.0:1789061131.117584:0:2406:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061131.117586:0:2406:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061131.117590:0:2406:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836ed80 x1875953296541568/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061131.117595:0:2406:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1789061131.117597:0:2406:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1789061131.117599:0:2406:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf4586b9c00. 02000000:00000001:2.0:1789061131.117603:0:2406:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1789061131.117605:0:2406:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:2.0:1789061131.117606:0:2406:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6800 refcount=3 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000020:00000001:2.0:1789061131.117608:0:2406:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:2.0:1789061131.117609:0:2406:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3100. 02000000:00000001:2.0:1789061131.117612:0:2406:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1789061131.117613:0:2406:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1789061131.117625:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836ed80. 00000100:00000001:2.0:1789061131.117629:0:2406:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1789061131.117630:0:2406:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061131.117633:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1789061131.117644:0:2406:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1789061131.117646:0:2406:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061131.117648:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1789061131.117650:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.118319:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061131.118324:0:2405:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1789061131.118330:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061131.118332:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061131.118335:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf450049c60. 00000400:00000200:3.0:1789061131.118340:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf442c1a7f8 00000400:00000010:3.0:1789061131.118343:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf442c1a7f8. 00000100:00000001:3.0:1789061131.118346:0:2405:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061131.118355:0:2405:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061131.118364:0:2405:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.118371:0:2405:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061131.118373:0:2405:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.118375:0:2405:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061131.118377:0:2405:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061131.118378:0:2405:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061131.118380:0:2405:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061131.118382:0:2405:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061131.118388:0:2405:0:(import.c:2196:obd_at_measure()) add 60 to ffff9bf448dc5460 time=135 v=60 (60 60 60 60) 00000100:00001000:3.0:1789061131.118392:0:2405:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc5418 time=112 v=5 (1 1 1 1) 00000100:00000001:3.0:1789061131.118394:0:2405:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061131.118404:0:2405:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061131.118406:0:2405:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789061131.118409:0:2405:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.118412:0:2405:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789061131.118414:0:2405:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9bf4804ce000: skip recheck: last_committed 30064771668 00000100:00000001:3.0:1789061131.118415:0:2405:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789061131.118416:0:2405:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.118421:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.118427:0:2405:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061131.118430:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061131.118431:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061131.118435:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061131.118441:0:2405:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541440:192.168.204.129@tcp:400:kworker.0 00000400:00000001:3.0:1789061131.118447:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1789061131.118448:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1789061131.118450:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf4500496e0. 00000400:00000200:3.0:1789061131.118453:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf442c1a990 00000400:00000010:3.0:1789061131.118454:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf442c1a990. 00000100:00000001:3.0:1789061131.118457:0:2405:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061131.118460:0:2405:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1789061131.118467:0:2405:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.118473:0:2405:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1789061131.118474:0:2405:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.118476:0:2405:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1789061131.118478:0:2405:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1789061131.118479:0:2405:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1789061131.118481:0:2405:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1789061131.118494:0:2405:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1789061131.118505:0:2405:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf448dc6460 time=135 v=5 (5 5 5 5) 00000100:00001000:3.0:1789061131.118508:0:2405:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6418 time=123 v=5 (1 1 0 0) 00000100:00000001:3.0:1789061131.118510:0:2405:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1789061131.118512:0:2405:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1789061131.118513:0:2405:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1789061131.118516:0:2405:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.118518:0:2405:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1789061131.118519:0:2405:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9bf4804ce000: skip recheck: last_committed 38654730077 00000100:00000001:3.0:1789061131.118521:0:2405:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1789061131.118523:0:2405:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1789061131.118527:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.118532:0:2405:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1789061131.118534:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1789061131.118535:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1789061131.118538:0:2405:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789061131.118557:0:2405:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf45836d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2405:1875953296541696:192.168.204.129@tcp:400:kworker.0 00000100:00000001:3.0:1789061131.118561:0:2405:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.118563:0:2405:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061131.118566:0:2405:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836fb80 x1875953296541440/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061216 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.118574:0:2405:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061131.118575:0:2405:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061131.118578:0:2405:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf451899a00. 02000000:00000001:3.0:1789061131.118582:0:2405:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061131.118585:0:2405:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061131.118587:0:2405:0:(genops.c:1222:class_import_put()) import ffff9bf448dc5000 refcount=8 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000020:00000001:3.0:1789061131.118589:0:2405:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061131.118591:0:2405:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3e00. 02000000:00000001:3.0:1789061131.119069:0:2405:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061131.119071:0:2405:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061131.119082:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836fb80. 00000100:00000001:3.0:1789061131.119086:0:2405:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061131.119087:0:2405:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.119089:0:2405:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1789061131.119096:0:2405:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf45836d500 x1875953296541696/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061147 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061131.119103:0:2405:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1789061131.119105:0:2405:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1789061131.119106:0:2405:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf451899200. 02000000:00000001:3.0:1789061131.119109:0:2405:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1789061131.119111:0:2405:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:3.0:1789061131.119112:0:2405:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6000 refcount=3 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000020:00000001:3.0:1789061131.119114:0:2405:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:3.0:1789061131.119116:0:2405:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf4816f3500. 02000000:00000001:3.0:1789061131.119118:0:2405:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1789061131.119119:0:2405:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1789061131.119120:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf45836d500. 00000100:00000001:3.0:1789061131.119122:0:2405:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1789061131.119124:0:2405:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.119126:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1789061131.119134:0:2405:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1789061131.119136:0:2405:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1789061131.119139:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1789061131.119141:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061131.803176:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061131.803184:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061131.803190:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061131.803192:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061131.803218:0:244529:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061131.803222:0:244529:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061131.803225:0:244529:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061132.826118:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061132.826128:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061132.826137:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061132.826138:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061132.826194:0:237353:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061132.826198:0:237353:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061132.826201:0:237353:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061133.850061:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061133.850070:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061133.850090:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061133.850091:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789061133.850392:0:253143:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789061133.850396:0:253143:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789061133.850399:0:253143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061134.874245:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061134.874255:0:228762:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1789061134.874258:0:228762:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1789061134.874262:0:228762:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061134.874268:0:228762:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1789061134.874270:0:228762:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061134.874272:0:228762:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061134.874275:0:228762:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061134.874279:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061134.874280:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061134.874353:0:237352:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061134.874357:0:237352:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061134.874359:0:237352:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1789061134.939436:0:276965:0:(connection.c:75:cpu_latency_work()) work item of ffff9bf44883f330 (cpu 3) has reached its deadline 4307515102, at 4307515137 00000100:00000010:3.0:1789061134.939459:0:276965:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9bf479a685a0. 00000100:00000040:2.0:1789061134.942914:0:273614:0:(connection.c:75:cpu_latency_work()) work item of ffff9bf44883f220 (cpu 2) has reached its deadline 4307515102, at 4307515140 00000100:00000010:2.0:1789061134.942934:0:273614:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9bf479a68a20. 00010000:00000001:0.0:1789061135.900049:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061135.900056:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061135.900062:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061135.900063:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061135.900085:0:249018:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061135.900088:0:249018:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061135.900105:0:249018:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1789061136.224728:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789061136.224741:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789061136.224753:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf47904d880. 00000020:00000040:1.0:1789061136.224760:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf442d05800 refcount=4 obd=MGC192.168.204.129@tcp 00000100:00000001:1.0:1789061136.224765:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789061136.224768:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789061136.224774:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789061136.224783:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf447c01400. 00000100:00000001:1.0:1789061136.224790:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.224803:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b->MGS req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789061136.224815:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789061136.224819:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.224825:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f26f2e7e] to pc [ptlrpcd_00_00+0] req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.224872:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.224876:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.224879:0:2403:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1789061136.224885:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.224893:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1789061136.224895:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.224899:0:2403:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1789061136.224902:0:2403:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789061136.224904:0:2403:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789061136.224908:0:2403:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf47904d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2403:1875953296541824:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.224912:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1789061136.224918:0:2403:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1789061136.224920:0:2403:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.224923:0:2403:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1789061136.224928:0:2403:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf4711ad200. 02000000:00000001:0.0:1789061136.224930:0:2403:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1789061136.224937:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:0.0:1789061136.224942:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf444d2c420. 00000400:00000010:0.0:1789061136.225015:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cfb28. 00000100:00000200:0.0:1789061136.225024:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541824, portal 25 00000100:00000001:0.0:1789061136.225027:0:2403:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1789061136.225029:0:2403:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634073037527168 : -110000672024448 : ffff9bf47904d880) 00000100:00000040:0.0:1789061136.225036:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.225043:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789061136.225046:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:0.0:1789061136.225048:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cf990. 00000100:00000200:0.0:1789061136.225052:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875953296541824, offset 0 00000400:00000200:0.0:1789061136.225058:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:0.0:1789061136.225069:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.225151:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:0.0:1789061136.225236:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1789061136.225241:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:0.0:1789061136.225245:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1789061136.225248:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:0.0:1789061136.225252:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554656:554656:256:4294967295] 192.168.204.129@tcp LPNI seq info [554607:554607:8:4294967295] 00000400:00000200:0.0:1789061136.225257:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.225267:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:0.0:1789061136.225272:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:0.0:1789061136.225278:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf4471b1600. 00000800:00000200:0.0:1789061136.225285:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:0.0:1789061136.225293:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:0.0:1789061136.225299:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf4471b1600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789061136.225353:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1789061136.225358:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:0.0:1789061136.225361:0:2403:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 2 is 4307520223 00000100:00000010:0.0:1789061136.225366:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf44712b780. 00000100:00000001:0.0:1789061136.225375:0:2403:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1789061136.225381:0:2403:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS new deadline estimation for cpu 3 is 4307520223 00000100:00000010:0.0:1789061136.225384:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9bf44712bd80. 00000100:00000001:0.0:1789061136.225387:0:2403:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.204.129@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1789061136.225390:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.225394:0:2403:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.225397:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.225400:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789061136.225456:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.225460:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789061136.225464:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789061136.225468:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf47904c380. 00000020:00000040:1.0:1789061136.225471:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf448dc5000 refcount=9 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000100:00000001:1.0:1789061136.225474:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789061136.225475:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789061136.225478:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789061136.225482:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf447c01000. 00000100:00000001:1.0:1789061136.225486:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.225492:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0000_UUID req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789061136.225510:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789061136.225514:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.225519:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000031de5b3] to pc [ptlrpcd_00_01+1] req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1789061136.229029:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061136.229040:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf4471b1600. 00000400:00000200:2.0:1789061136.229049:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.229058:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061136.229067:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cf990 00000400:00000010:2.0:1789061136.229071:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cf990. 00000100:00000001:2.0:1789061136.229089:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061136.229098:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.229113:0:2398:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061136.229118:0:2398:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.229126:0:2398:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061136.229128:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1789061136.229714:0:2403:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1789061136.229718:0:2403:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1789061136.229724:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.229726:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.229729:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789061136.229733:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1789061136.229735:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.229737:0:2403:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1789061136.229747:0:2403:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.229749:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.229751:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.229752:0:2403:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1789061136.229761:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.229769:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1789061136.229772:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.229776:0:2403:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1789061136.229779:0:2403:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789061136.229782:0:2403:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789061136.229785:0:2403:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf47904c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2403:1875953296541952:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.229790:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1789061136.229795:0:2403:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1789061136.229798:0:2403:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.229800:0:2403:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1789061136.229807:0:2403:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf4711acc00. 02000000:00000001:0.0:1789061136.229810:0:2403:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1789061136.229817:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:0.0:1789061136.229821:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf444d2c5d8. 00000400:00000010:0.0:1789061136.229827:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cf198. 00000100:00000200:0.0:1789061136.229833:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296541952, portal 10 00000100:00000001:0.0:1789061136.229836:0:2403:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1789061136.229837:0:2403:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634073037521792 : -110000672029824 : ffff9bf47904c380) 00000100:00000040:0.0:1789061136.229846:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.229853:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789061136.229856:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:0.0:1789061136.229858:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cf4c8. 00000100:00000200:0.0:1789061136.229861:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296541952, offset 0 00000400:00000200:0.0:1789061136.229867:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:0.0:1789061136.229875:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.229883:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:0.0:1789061136.229895:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1789061136.229900:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:0.0:1789061136.229904:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1789061136.229906:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:0.0:1789061136.229910:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554657:554657:256:4294967295] 192.168.204.129@tcp LPNI seq info [554608:554608:8:4294967295] 00000400:00000200:0.0:1789061136.229915:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.229925:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:0.0:1789061136.229931:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:0.0:1789061136.229936:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf4471b1400. 00000800:00000200:0.0:1789061136.229943:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:0.0:1789061136.229973:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:0.0:1789061136.229979:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf4471b1400 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789061136.229993:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789061136.229998:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: using a timeout of 29468 usecs (30 jiffies) 00000400:00000200:0.0:1789061136.230002:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:0.0:1789061136.230004:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307516457 00000100:00000001:0.0:1789061136.230008:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307520223 00000100:00000001:0.0:1789061136.230010:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307516457 00000100:00000001:0.0:1789061136.230013:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307520223 00000100:00000001:0.0:1789061136.230015:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.230018:0:2403:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.230020:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.230022:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789061136.230725:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230732:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789061136.230736:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789061136.230743:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf47904c000. 00000020:00000040:1.0:1789061136.230747:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6800 refcount=4 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000100:00000001:1.0:1789061136.230751:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789061136.230753:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789061136.230757:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789061136.230764:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf447c01300. 00000100:00000001:1.0:1789061136.230770:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230778:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-MDT0001_UUID req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789061136.230789:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789061136.230793:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230798:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000788481af] to pc [ptlrpcd_00_00+0] req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789061136.230808:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230810:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1789061136.230813:0:250456:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1789061136.230816:0:250456:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9bf47904ca80. 00000020:00000040:1.0:1789061136.230818:0:250456:0:(genops.c:1209:class_import_get()) import ffff9bf448dc6000 refcount=4 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000100:00000001:1.0:1789061136.230820:0:250456:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1789061136.230821:0:250456:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1789061136.230822:0:250456:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1789061136.230825:0:250456:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9bf447c01e00. 00000100:00000001:1.0:1789061136.230828:0:250456:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230832:0:250456:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0000_UUID req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1789061136.230838:0:250456:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1789061136.230847:0:250456:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230851:0:250456:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cc2867f0] to pc [ptlrpcd_00_01+1] req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789061136.230859:0:250456:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.230861:0:250456:0:(pinger.c:229:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1789061136.230864:0:250456:0:(pinger.c:247:ptlrpc_pinger_process_import()) f9405128-a312-44b7-8d65-caa32bc1b7dc->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1789061136.230870:0:250456:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12854) 00000100:00000001:0.0:1789061136.230987:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.230989:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.230991:0:2403:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1789061136.230997:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.231004:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1789061136.231005:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.231014:0:2403:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1789061136.231017:0:2403:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789061136.231019:0:2403:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789061136.231022:0:2403:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf47904c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2403:1875953296542080:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.231026:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1789061136.231031:0:2403:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1789061136.231033:0:2403:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.231034:0:2403:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1789061136.231038:0:2403:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf445f66000. 02000000:00000001:0.0:1789061136.231040:0:2403:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1789061136.231045:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:0.0:1789061136.231048:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf444d2c8f0. 00000400:00000010:0.0:1789061136.231052:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cf2a8. 00000100:00000200:0.0:1789061136.231056:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296542080, portal 10 00000100:00000001:0.0:1789061136.231058:0:2403:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1789061136.231060:0:2403:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634073037520896 : -110000672030720 : ffff9bf47904c000) 00000100:00000040:0.0:1789061136.231065:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.231070:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789061136.231084:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:0.0:1789061136.231087:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cf5d8. 00000100:00000200:0.0:1789061136.231090:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875953296542080, offset 0 00000400:00000200:0.0:1789061136.231095:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:0.0:1789061136.231101:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.231106:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:0.0:1789061136.231111:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1789061136.231114:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:0.0:1789061136.231117:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1789061136.231120:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:0.0:1789061136.231123:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554658:554658:255:4294967295] 192.168.204.129@tcp LPNI seq info [554609:554609:7:4294967295] 00000400:00000200:0.0:1789061136.231127:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.231135:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:0.0:1789061136.231146:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:0.0:1789061136.231151:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf4471b1600. 00000800:00000200:0.0:1789061136.231156:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:0.0:1789061136.231283:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:0.0:1789061136.231289:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf4471b1600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789061136.231293:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789061136.231297:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: using a timeout of 22406 usecs (23 jiffies) 00000400:00000200:0.0:1789061136.231301:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:0.0:1789061136.231304:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307516451 00000100:00000001:0.0:1789061136.231307:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307520223 00000100:00000001:0.0:1789061136.231309:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307516451 00000100:00000001:0.0:1789061136.231311:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307520223 00000100:00000001:0.0:1789061136.231313:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.231316:0:2403:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.231318:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.231321:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.231327:0:2403:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1789061136.231329:0:2403:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1789061136.231331:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.231332:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.231333:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789061136.231335:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1789061136.231337:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.231339:0:2403:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1789061136.231340:0:2403:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.231342:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.231343:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.231345:0:2403:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1789061136.231349:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.231356:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1789061136.231357:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.231366:0:2403:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1789061136.231368:0:2403:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1789061136.231370:0:2403:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1789061136.231373:0:2403:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9bf47904ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2403:1875953296542208:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.231376:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1789061136.231378:0:2403:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1789061136.231380:0:2403:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.231382:0:2403:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1789061136.231385:0:2403:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9bf448b95000. 02000000:00000001:0.0:1789061136.231387:0:2403:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1789061136.231391:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000010:0.0:1789061136.231394:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9bf444d2c528. 00000400:00000010:0.0:1789061136.231397:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cf3b8. 00000100:00000200:0.0:1789061136.231400:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875953296542208, portal 4 00000100:00000001:0.0:1789061136.231402:0:2403:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1789061136.231403:0:2403:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446634073037523584 : -110000672028032 : ffff9bf47904ca80) 00000100:00000040:0.0:1789061136.231409:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.231415:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1789061136.231417:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.129@tcp 00000400:00000010:0.0:1789061136.231420:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9bf4471cfcc0. 00000100:00000200:0.0:1789061136.231422:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875953296542208, offset 0 00000400:00000200:0.0:1789061136.231426:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.129@tcp 00000400:00000200:0.0:1789061136.231431:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.231434:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.129@tcp local destination 00000400:00000200:0.0:1789061136.231437:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1789061136.231440:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.29@tcp 00000400:00000200:0.0:1789061136.231442:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.29@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1789061136.231456:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.129@tcp 00000400:00000200:0.0:1789061136.231459:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.29@tcp NI seq info: [554659:554659:254:4294967295] 192.168.204.129@tcp LPNI seq info [554610:554610:6:4294967295] 00000400:00000200:0.0:1789061136.231463:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:0.0:1789061136.231468:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp:) -> 192.168.204.129@tcp(192.168.204.129@tcp:192.168.204.129@tcp) : PUT try# 0 00000800:00000200:0.0:1789061136.231479:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.129@tcp 00000800:00000010:0.0:1789061136.231482:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9bf4471b1300. 00000800:00000200:0.0:1789061136.231486:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9bf442d389c0] -> 12345-192.168.204.129@tcp (5) 00000800:00000200:0.0:1789061136.231490:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.129@tcp ip 192.168.204.129:988 00000800:00000200:0.0:1789061136.231492:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9bf4471b1300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1789061136.231495:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789061136.231624:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: using a timeout of 111032 usecs (112 jiffies) 00000400:00000200:0.0:1789061136.231629:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000100:00000001:0.0:1789061136.231631:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 2 is 4307516540 00000100:00000001:0.0:1789061136.231711:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 2, keep current deadline 4307520223 00000100:00000001:0.0:1789061136.231714:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS new deadline estimation for cpu 3 is 4307516540 00000100:00000001:0.0:1789061136.231715:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9bf4804ce000: PM QoS request active for cpu 3, keep current deadline 4307520223 00000100:00000001:0.0:1789061136.231717:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.231719:0:2403:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.231722:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.231724:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1789061136.231840:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061136.231844:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf4471b1400. 00000400:00000200:2.0:1789061136.231850:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.231855:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061136.231860:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cf4c8 00000400:00000010:2.0:1789061136.231863:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cf4c8. 00000100:00000001:2.0:1789061136.231867:0:2397:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061136.231873:0:2397:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.231883:0:2397:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061136.231886:0:2397:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.231892:0:2397:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061136.231894:0:2397:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789061136.231907:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061136.231909:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf4471b1600. 00000400:00000200:2.0:1789061136.231918:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.231922:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061136.231925:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cf5d8 00000400:00000010:2.0:1789061136.231927:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cf5d8. 00000100:00000001:2.0:1789061136.231930:0:2397:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061136.231934:0:2397:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.231940:0:2397:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061136.231943:0:2397:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.231977:0:2397:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061136.231979:0:2397:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:2.0:1789061136.232057:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1789061136.232060:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9bf4471b1300. 00000400:00000200:2.0:1789061136.232063:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.232066:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000400:00000200:2.0:1789061136.232069:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cfcc0 00000400:00000010:2.0:1789061136.232081:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cfcc0. 00000100:00000001:2.0:1789061136.232084:0:2397:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:2.0:1789061136.232087:0:2397:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.232095:0:2397:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1789061136.232098:0:2397:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.232104:0:2397:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1789061136.232105:0:2397:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1789061136.234472:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.234492:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061136.234510:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.234514:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061136.234524:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.234531:0:2398:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061136.234541:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.234555:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.234561:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.234565:0:2398:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 25 MB=0x6aa2b69db0880 00000400:00000200:2.0:1789061136.234570:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.234577:0:2398:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.129@tcp of length 224/224 into md 0x5348a5 [1] + 224 00000800:00000001:2.0:1789061136.234585:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.234593:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061136.234597:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.234601:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061136.234609:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061136.234618:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061136.234628:0:2398:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.234651:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061136.234660:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.234664:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.235204:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.235210:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1789061136.235219:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1789061136.235222:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1789061136.235226:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf444d2c420. 00000400:00000200:0.0:1789061136.235231:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cfb28 00000400:00000010:0.0:1789061136.235234:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cfb28. 00000100:00000001:0.0:1789061136.235237:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1789061136.235245:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1789061136.235254:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.235260:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1789061136.235262:0:2403:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.235265:0:2403:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1789061136.235274:0:2403:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1789061136.235276:0:2403:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789061136.235279:0:2403:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.235281:0:2403:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789061136.235288:0:2403:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf442d05c60 time=140 v=5 (5 5 5 5) 00000100:00001000:0.0:1789061136.235292:0:2403:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf442d05c18 time=119 v=5 (1 1 1 1) 00000100:00000001:0.0:1789061136.235295:0:2403:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1789061136.235297:0:2403:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061136.235300:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1789061136.235301:0:2403:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.235304:0:2403:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789061136.235310:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.235317:0:2403:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.235319:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1789061136.235322:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1789061136.235325:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789061136.235332:0:2403:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf47904d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f5ab2bd2-b9ea-4859-ace5-e36943a7bf2b:2403:1875953296541824:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.235338:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.235340:0:2403:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1789061136.235343:0:2403:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf47904d880 x1875953296541824/t0(0) o400->MGC192.168.204.129@tcp@192.168.204.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.235350:0:2403:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1789061136.235353:0:2403:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1789061136.235356:0:2403:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf4711ad200. 02000000:00000001:0.0:1789061136.235362:0:2403:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1789061136.235364:0:2403:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:0.0:1789061136.235366:0:2403:0:(genops.c:1222:class_import_put()) import ffff9bf442d05800 refcount=3 obd=MGC192.168.204.129@tcp 00000020:00000001:0.0:1789061136.235368:0:2403:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:0.0:1789061136.235371:0:2403:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf447c01400. 02000000:00000001:0.0:1789061136.235374:0:2403:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1789061136.235376:0:2403:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1789061136.235383:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf47904d880. 00000100:00000001:0.0:1789061136.235386:0:2403:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1789061136.235387:0:2403:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.235390:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.235392:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.235394:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.235395:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.235397:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789061136.236263:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.236277:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061136.236280:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.236283:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061136.236290:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236296:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061136.236304:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236308:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236313:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236317:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0900 00000400:00000200:2.0:1789061136.236322:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236327:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x5348ad [1] + 224 00000800:00000001:2.0:1789061136.236334:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.236338:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061136.236341:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.236351:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061136.236357:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061136.236365:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061136.236375:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.236394:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061136.236401:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.236406:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1789061136.236407:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.236409:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061136.236412:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236416:0:2397:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:2.0:1789061136.236420:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236422:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236424:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236427:0:2397:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 10 MB=0x6aa2b69db0980 00000400:00000200:2.0:1789061136.236430:0:2397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:2.0:1789061136.236434:0:2397:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.129@tcp of length 224/224 into md 0x5348b5 [1] + 224 00000800:00000001:2.0:1789061136.236437:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1789061136.236963:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.236966:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1789061136.236972:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1789061136.236974:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1789061136.236977:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf444d2c5d8. 00000400:00000200:0.0:1789061136.236982:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cf198 00000400:00000010:0.0:1789061136.236984:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cf198. 00000100:00000001:0.0:1789061136.236987:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1789061136.236994:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1789061136.237003:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.237010:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1789061136.237012:0:2403:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.237015:0:2403:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1789061136.237017:0:2403:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1789061136.237020:0:2403:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789061136.237022:0:2403:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.237024:0:2403:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789061136.237035:0:2403:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf448dc5460 time=140 v=60 (60 60 60 60) 00000100:00001000:0.0:1789061136.237039:0:2403:0:(import.c:2262:obd_at_measure()) AT ffff9bf448dc5460 change: old=60 new=5 delta=-55 (val=5) hist 60 60 60 60 00000100:00001000:0.0:1789061136.237043:0:2403:0:(client.c:430:ptlrpc_at_adj_service()) The RPC service estimate for lustre-MDT0000-mdc-ffff9bf4804ce000 ptl 12 has changed from 60 to 5 00000100:00001000:0.0:1789061136.237046:0:2403:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc5418 time=117 v=5 (1 1 1 1) 00000100:00000001:0.0:1789061136.237048:0:2403:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1789061136.237050:0:2403:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061136.237053:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1789061136.237056:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237061:0:2403:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1789061136.237070:0:2403:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9bf4804ce000: skip recheck: last_committed 30064771668 00000100:00000001:0.0:1789061136.237083:0:2403:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1789061136.237086:0:2403:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789061136.237092:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.237099:0:2403:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.237102:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1789061136.237104:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1789061136.237108:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789061136.237115:0:2403:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf47904c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2403:1875953296541952:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.237121:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237124:0:2403:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1789061136.237127:0:2403:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf47904c380 x1875953296541952/t0(0) o400->lustre-MDT0000-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.237132:0:2403:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1789061136.237135:0:2403:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1789061136.237138:0:2403:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf4711acc00. 02000000:00000001:0.0:1789061136.237142:0:2403:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1789061136.237144:0:2403:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:0.0:1789061136.237146:0:2403:0:(genops.c:1222:class_import_put()) import ffff9bf448dc5000 refcount=8 obd=lustre-MDT0000-mdc-ffff9bf4804ce000 00000020:00000001:0.0:1789061136.237148:0:2403:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:0.0:1789061136.237150:0:2403:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf447c01000. 02000000:00000001:0.0:1789061136.237152:0:2403:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1789061136.237154:0:2403:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1789061136.237156:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf47904c380. 00000100:00000001:0.0:1789061136.237159:0:2403:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1789061136.237160:0:2403:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.237173:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237175:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.237176:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.237177:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237179:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1789061136.237306:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1789061136.237310:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1789061136.237314:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:2.0:1789061136.237319:0:2397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:2.0:1789061136.237324:0:2397:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1789061136.237333:0:2397:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1789061136.237351:0:2397:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:2.0:1789061136.237357:0:2397:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1789061136.237369:0:2397:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.237464:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.237466:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1789061136.237469:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1789061136.237470:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1789061136.237473:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf444d2c8f0. 00000400:00000200:0.0:1789061136.237475:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cf2a8 00000400:00000010:0.0:1789061136.237477:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cf2a8. 00000100:00000001:0.0:1789061136.237479:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1789061136.237483:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1789061136.237491:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.237496:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1789061136.237498:0:2403:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.237499:0:2403:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1789061136.237501:0:2403:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1789061136.237622:0:2403:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1789061136.237624:0:2403:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1789061136.237626:0:2403:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1789061136.237631:0:2403:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf448dc6c60 time=140 v=60 (60 60 60 60) 00000100:00001000:0.0:1789061136.237704:0:2403:0:(import.c:2262:obd_at_measure()) AT ffff9bf448dc6c60 change: old=60 new=5 delta=-55 (val=5) hist 60 60 60 60 00000100:00001000:0.0:1789061136.237707:0:2403:0:(client.c:430:ptlrpc_at_adj_service()) The RPC service estimate for lustre-MDT0001-mdc-ffff9bf4804ce000 ptl 12 has changed from 60 to 5 00000100:00001000:0.0:1789061136.237710:0:2403:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6c18 time=93 v=5 (1 1 1 1) 00000100:00000001:0.0:1789061136.237712:0:2403:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1789061136.237713:0:2403:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061136.237715:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1789061136.237720:0:2403:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1789061136.237725:0:2403:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237728:0:2403:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1789061136.237729:0:2403:0:(client.c:3122:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9bf4804ce000: skip recheck: last_committed 21474836545 00000100:00000001:0.0:1789061136.237731:0:2403:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1789061136.237732:0:2403:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1789061136.237735:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.237748:0:2403:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1789061136.237750:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1789061136.237752:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1789061136.237756:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789061136.237763:0:2403:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf47904c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2403:1875953296542080:192.168.204.129@tcp:400:kworker.0 00000100:00000001:0.0:1789061136.237767:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237769:0:2403:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1789061136.237772:0:2403:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf47904c000 x1875953296542080/t0(0) o400->lustre-MDT0001-mdc-ffff9bf4804ce000@192.168.204.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1789061221 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1789061136.237778:0:2403:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1789061136.237779:0:2403:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1789061136.237781:0:2403:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf445f66000. 02000000:00000001:0.0:1789061136.237785:0:2403:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1789061136.237786:0:2403:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:0.0:1789061136.237788:0:2403:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6800 refcount=3 obd=lustre-MDT0001-mdc-ffff9bf4804ce000 00000020:00000001:0.0:1789061136.237790:0:2403:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:0.0:1789061136.237792:0:2403:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf447c01300. 02000000:00000001:0.0:1789061136.237794:0:2403:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1789061136.237795:0:2403:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1789061136.237796:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf47904c000. 00000100:00000001:0.0:1789061136.237799:0:2403:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1789061136.237800:0:2403:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1789061136.237802:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237804:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1789061136.237805:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1789061136.237807:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1789061136.237808:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789061136.237920:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789061136.237923:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1789061136.245013:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061136.245032:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1789061136.245037:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061136.245041:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789061136.245053:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061136.245062:0:2398:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.29@tcp(192.168.204.29@tcp) <- 192.168.204.129@tcp : PUT - for me 00000400:00000200:3.0:1789061136.245083:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061136.245088:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061136.245094:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061136.245099:0:2398:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.129@tcp of length 224 into portal 4 MB=0x6aa2b69db0a00 00000400:00000200:3.0:1789061136.245115:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.204.129@tcp to cpt 1 00000400:00000200:3.0:1789061136.245123:0:2398:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.129@tcp of length 224/224 into md 0x5348bd [1] + 224 00000800:00000001:3.0:1789061136.245132:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061136.245145:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1789061136.245149:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1789061136.245155:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.29@tcp->192.168.204.129@tcp: PUT: OK 00000100:00000001:3.0:1789061136.245164:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:3.0:1789061136.245240:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1789061136.245253:0:2398:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1789061136.245268:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:3.0:1789061136.245278:0:2398:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1789061136.245282:0:2398:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789061136.245350:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789061136.245355:0:2403:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1789061136.245364:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1789061136.245366:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1789061136.245369:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9bf444d2c528. 00000400:00000200:1.0:1789061136.245375:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9bf4471cf3b8 00000400:00000010:1.0:1789061136.245379:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9bf4471cf3b8. 00000100:00000001:1.0:1789061136.245382:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:1.0:1789061136.245387:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1789061136.245396:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789061136.245402:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1789061136.245404:0:2403:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789061136.245407:0:2403:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:1.0:1789061136.245410:0:2403:0:(sec.c:1131:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1789061136.245412:0:2403:0:(pack_generic.c:607:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1789061136.245415:0:2403:0:(pack_generic.c:625:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1789061136.245418:0:2403:0:(sec.c:1193:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1789061136.245428:0:2403:0:(import.c:2196:obd_at_measure()) add 5 to ffff9bf448dc6460 time=140 v=5 (5 5 5 5) 00000100:00001000:1.0:1789061136.245432:0:2403:0:(import.c:2196:obd_at_measure()) add 1 to ffff9bf448dc6418 time=128 v=5 (1 1 0 0) 00000100:00000001:1.0:1789061136.245435:0:2403:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1789061136.245436:0:2403:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1789061136.245439:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1789061136.245461:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789061136.245465:0:2403:0:(client.c:3110:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1789061136.245468:0:2403:0:(client.c:3122:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9bf4804ce000: skip recheck: last_committed 38654730077 00000100:00000001:1.0:1789061136.245471:0:2403:0:(client.c:3123:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1789061136.245472:0:2403:0:(client.c:1798:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1789061136.245478:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789061136.245485:0:2403:0:(client.c:2355:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1789061136.245488:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1789061136.245491:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1789061136.245494:0:2403:0:(lustre_net.h:2418:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789061136.245717:0:2403:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9bf47904ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f9405128-a312-44b7-8d65-caa32bc1b7dc:2403:1875953296542208:192.168.204.129@tcp:400:kworker.0 00000100:00000001:1.0:1789061136.245723:0:2403:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789061136.245726:0:2403:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1789061136.245730:0:2403:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9bf47904ca80 x1875953296542208/t0(0) o400->lustre-OST0000-osc-ffff9bf4804ce000@192.168.204.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1789061152 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1789061136.245737:0:2403:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1789061136.245739:0:2403:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1789061136.245744:0:2403:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9bf448b95000. 02000000:00000001:1.0:1789061136.245750:0:2403:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1789061136.245753:0:2403:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000040:1.0:1789061136.245755:0:2403:0:(genops.c:1222:class_import_put()) import ffff9bf448dc6000 refcount=3 obd=lustre-OST0000-osc-ffff9bf4804ce000 00000020:00000001:1.0:1789061136.245758:0:2403:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000010:1.0:1789061136.245760:0:2403:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9bf447c01e00. 02000000:00000001:1.0:1789061136.245764:0:2403:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1789061136.245767:0:2403:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1789061136.245769:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9bf47904ca80. 00000100:00000001:1.0:1789061136.245772:0:2403:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1789061136.245774:0:2403:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789061136.245776:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1789061136.245784:0:2403:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1789061136.245792:0:2403:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1789061136.245794:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1789061136.245796:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061136.922226:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061136.922233:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061136.922238:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061136.922239:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061136.922272:0:244529:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061136.922277:0:244529:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061136.922282:0:244529:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1789061137.454795:0:286143:0:(llite_lib.c:4258:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1789061137.843864:0:286180:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:2.0:1789061137.843872:0:286180:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity test_816: @@@@@@ FAIL: client: import is not in IDLE state after 50 00000001:00000001:2.0:1789061137.852749:0:286180:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00010000:00000001:0.0:1789061137.949438:0:283232:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061137.949445:0:283232:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061137.949451:0:283232:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061137.949452:0:283232:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1789061137.949489:0:237353:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1789061137.949492:0:237353:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1789061137.949495:0:237353:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1789061138.972588:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061138.972596:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1789061138.972603:0:228762:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1789061138.972604:0:228762:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1789061138.975917:0:253143:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1789061138.975924:0:253143:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1789061138.975929:0:253143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:3.0:1789061139.584822:0:286280:0:(llite_lib.c:4258:ll_show_options()) Process leaving (rc=0 : 0 : 0) Debug log: 2673 lines, 2673 kept, 0 dropped, 0 bad.